2024-11-15 00:44:24,777 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 00:44:24,793 main DEBUG Took 0.013643 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 00:44:24,794 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 00:44:24,794 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 00:44:24,796 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 00:44:24,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,807 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 00:44:24,825 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,828 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,829 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,831 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,832 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,833 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,834 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,834 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,835 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,836 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,837 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,837 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,838 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,838 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,839 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,839 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,840 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,840 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,841 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:44:24,841 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,842 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 00:44:24,844 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:44:24,845 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 00:44:24,848 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 00:44:24,849 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 00:44:24,850 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 00:44:24,851 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 00:44:24,863 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 00:44:24,867 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 00:44:24,869 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 00:44:24,870 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 00:44:24,870 main DEBUG createAppenders(={Console}) 2024-11-15 00:44:24,871 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-15 00:44:24,872 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 00:44:24,872 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-15 00:44:24,873 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 00:44:24,873 main DEBUG OutputStream closed 2024-11-15 00:44:24,874 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 00:44:24,874 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 00:44:24,875 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-15 00:44:24,984 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 00:44:24,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 00:44:24,990 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 00:44:24,992 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 00:44:24,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 00:44:24,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 00:44:24,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 00:44:24,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 00:44:24,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 00:44:24,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 00:44:24,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 00:44:24,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 00:44:24,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 00:44:24,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 00:44:24,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 00:44:24,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 00:44:24,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 00:44:24,999 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 00:44:25,002 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 00:44:25,002 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-15 00:44:25,003 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 00:44:25,004 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-15T00:44:25,356 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d 2024-11-15 00:44:25,364 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 00:44:25,364 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T00:44:25,384 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-15T00:44:25,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T00:44:25,470 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/cluster_c94eddf5-b65f-7607-3f6e-7f317580d804, deleteOnExit=true 2024-11-15T00:44:25,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T00:44:25,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/test.cache.data in system properties and HBase conf 2024-11-15T00:44:25,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T00:44:25,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/hadoop.log.dir in system properties and HBase conf 2024-11-15T00:44:25,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T00:44:25,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T00:44:25,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T00:44:25,603 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T00:44:25,726 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T00:44:25,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T00:44:25,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T00:44:25,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T00:44:25,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T00:44:25,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T00:44:25,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T00:44:25,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T00:44:25,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T00:44:25,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T00:44:25,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/nfs.dump.dir in system properties and HBase conf 2024-11-15T00:44:25,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/java.io.tmpdir in system properties and HBase conf 2024-11-15T00:44:25,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T00:44:25,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T00:44:25,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T00:44:26,916 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T00:44:27,020 INFO [Time-limited test {}] log.Log(170): Logging initialized @3193ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T00:44:27,114 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T00:44:27,236 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T00:44:27,280 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T00:44:27,281 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T00:44:27,283 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T00:44:27,300 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T00:44:27,305 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/hadoop.log.dir/,AVAILABLE} 2024-11-15T00:44:27,307 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T00:44:27,586 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/java.io.tmpdir/jetty-localhost-43119-hadoop-hdfs-3_4_1-tests_jar-_-any-8504680278468019013/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T00:44:27,597 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43119} 2024-11-15T00:44:27,597 INFO [Time-limited test {}] server.Server(415): Started @3770ms 2024-11-15T00:44:28,808 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T00:44:28,819 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T00:44:28,822 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T00:44:28,822 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T00:44:28,822 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T00:44:28,823 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/hadoop.log.dir/,AVAILABLE} 2024-11-15T00:44:28,824 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T00:44:28,939 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/java.io.tmpdir/jetty-localhost-36119-hadoop-hdfs-3_4_1-tests_jar-_-any-16458310775426964709/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T00:44:28,940 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:36119} 2024-11-15T00:44:28,940 INFO [Time-limited test {}] server.Server(415): Started @5114ms 2024-11-15T00:44:29,005 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T00:44:30,159 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/cluster_c94eddf5-b65f-7607-3f6e-7f317580d804/data/data1/current/BP-1642220532-172.17.0.3-1731631466348/current, will proceed with Du for space computation calculation, 2024-11-15T00:44:30,159 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/cluster_c94eddf5-b65f-7607-3f6e-7f317580d804/data/data2/current/BP-1642220532-172.17.0.3-1731631466348/current, will proceed with Du for space computation calculation, 2024-11-15T00:44:30,205 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T00:44:30,267 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfec94ddafb7f8cff with lease ID 0x40f92fbfcf06cb36: Processing first storage report for DS-b52a2a7a-806c-4b77-aaee-5676bb37066d from datanode DatanodeRegistration(127.0.0.1:38979, datanodeUuid=7fa1af05-92e5-4b5f-9f70-a44c27aaea50, infoPort=38505, infoSecurePort=0, ipcPort=45675, storageInfo=lv=-57;cid=testClusterID;nsid=2009897078;c=1731631466348) 2024-11-15T00:44:30,270 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfec94ddafb7f8cff with lease ID 0x40f92fbfcf06cb36: from storage DS-b52a2a7a-806c-4b77-aaee-5676bb37066d node DatanodeRegistration(127.0.0.1:38979, datanodeUuid=7fa1af05-92e5-4b5f-9f70-a44c27aaea50, infoPort=38505, infoSecurePort=0, ipcPort=45675, storageInfo=lv=-57;cid=testClusterID;nsid=2009897078;c=1731631466348), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-15T00:44:30,270 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfec94ddafb7f8cff with lease ID 0x40f92fbfcf06cb36: Processing first storage report for DS-ce18cfdd-427a-4ae5-86c0-f42e9c5bdb05 from datanode DatanodeRegistration(127.0.0.1:38979, datanodeUuid=7fa1af05-92e5-4b5f-9f70-a44c27aaea50, infoPort=38505, infoSecurePort=0, ipcPort=45675, storageInfo=lv=-57;cid=testClusterID;nsid=2009897078;c=1731631466348) 2024-11-15T00:44:30,271 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfec94ddafb7f8cff with lease ID 0x40f92fbfcf06cb36: from storage DS-ce18cfdd-427a-4ae5-86c0-f42e9c5bdb05 node DatanodeRegistration(127.0.0.1:38979, datanodeUuid=7fa1af05-92e5-4b5f-9f70-a44c27aaea50, infoPort=38505, infoSecurePort=0, ipcPort=45675, storageInfo=lv=-57;cid=testClusterID;nsid=2009897078;c=1731631466348), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T00:44:30,334 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d 2024-11-15T00:44:30,435 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/cluster_c94eddf5-b65f-7607-3f6e-7f317580d804/zookeeper_0, clientPort=62640, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/cluster_c94eddf5-b65f-7607-3f6e-7f317580d804/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/cluster_c94eddf5-b65f-7607-3f6e-7f317580d804/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T00:44:30,448 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62640 2024-11-15T00:44:30,462 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:44:30,465 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:44:30,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741825_1001 (size=7) 2024-11-15T00:44:31,171 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403 with version=8 2024-11-15T00:44:31,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/hbase-staging 2024-11-15T00:44:31,258 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T00:44:31,482 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fb2f6cb7846a:0 server-side Connection retries=45 2024-11-15T00:44:31,492 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:44:31,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T00:44:31,497 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T00:44:31,497 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:44:31,498 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T00:44:31,668 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T00:44:31,749 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T00:44:31,762 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T00:44:31,767 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T00:44:31,797 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7378 (auto-detected) 2024-11-15T00:44:31,798 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T00:44:31,826 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33521 2024-11-15T00:44:31,861 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33521 connecting to ZooKeeper ensemble=127.0.0.1:62640 2024-11-15T00:44:31,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:335210x0, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T00:44:31,975 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33521-0x1013bf352ec0000 connected 2024-11-15T00:44:32,091 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:44:32,093 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:44:32,103 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:44:32,108 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403, hbase.cluster.distributed=false 2024-11-15T00:44:32,137 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T00:44:32,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33521 2024-11-15T00:44:32,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33521 2024-11-15T00:44:32,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33521 2024-11-15T00:44:32,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33521 2024-11-15T00:44:32,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33521 2024-11-15T00:44:32,239 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fb2f6cb7846a:0 server-side Connection retries=45 2024-11-15T00:44:32,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:44:32,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T00:44:32,241 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T00:44:32,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:44:32,242 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T00:44:32,244 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T00:44:32,247 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T00:44:32,247 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34537 2024-11-15T00:44:32,249 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34537 connecting to ZooKeeper ensemble=127.0.0.1:62640 2024-11-15T00:44:32,250 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:44:32,254 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:44:32,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:345370x0, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T00:44:32,269 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:345370x0, quorum=127.0.0.1:62640, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:44:32,269 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34537-0x1013bf352ec0001 connected 2024-11-15T00:44:32,273 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T00:44:32,280 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T00:44:32,283 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T00:44:32,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T00:44:32,289 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34537 2024-11-15T00:44:32,290 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34537 2024-11-15T00:44:32,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34537 2024-11-15T00:44:32,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34537 2024-11-15T00:44:32,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34537 2024-11-15T00:44:32,306 DEBUG [M:0;fb2f6cb7846a:33521 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fb2f6cb7846a:33521 2024-11-15T00:44:32,307 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fb2f6cb7846a,33521,1731631471305 2024-11-15T00:44:32,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:44:32,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:44:32,323 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fb2f6cb7846a,33521,1731631471305 2024-11-15T00:44:32,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:32,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T00:44:32,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:32,353 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T00:44:32,354 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fb2f6cb7846a,33521,1731631471305 from backup master directory 2024-11-15T00:44:32,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:44:32,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fb2f6cb7846a,33521,1731631471305 2024-11-15T00:44:32,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:44:32,364 WARN [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T00:44:32,364 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fb2f6cb7846a,33521,1731631471305 2024-11-15T00:44:32,366 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T00:44:32,367 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T00:44:32,424 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/hbase.id] with ID: 78911300-2cd1-4362-aa5b-886d0f47db71 2024-11-15T00:44:32,424 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/.tmp/hbase.id 2024-11-15T00:44:32,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741826_1002 (size=42) 2024-11-15T00:44:32,837 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/.tmp/hbase.id]:[hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/hbase.id] 2024-11-15T00:44:32,973 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:44:32,980 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T00:44:33,008 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-15T00:44:33,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:33,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:33,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741827_1003 (size=196) 2024-11-15T00:44:33,522 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:44:33,524 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T00:44:33,538 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:33,542 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T00:44:33,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741828_1004 (size=1189) 2024-11-15T00:44:33,989 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store 2024-11-15T00:44:34,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741829_1005 (size=34) 2024-11-15T00:44:34,415 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T00:44:34,419 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:44:34,421 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T00:44:34,421 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:44:34,421 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:44:34,423 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T00:44:34,424 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:44:34,424 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:44:34,426 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731631474421Disabling compacts and flushes for region at 1731631474421Disabling writes for close at 1731631474424 (+3 ms)Writing region close event to WAL at 1731631474424Closed at 1731631474424 2024-11-15T00:44:34,432 WARN [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/.initializing 2024-11-15T00:44:34,433 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/WALs/fb2f6cb7846a,33521,1731631471305 2024-11-15T00:44:34,448 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T00:44:34,469 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fb2f6cb7846a%2C33521%2C1731631471305, suffix=, logDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/WALs/fb2f6cb7846a,33521,1731631471305, archiveDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/oldWALs, maxLogs=10 2024-11-15T00:44:34,500 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/WALs/fb2f6cb7846a,33521,1731631471305/fb2f6cb7846a%2C33521%2C1731631471305.1731631474474, exclude list is [], retry=0 2024-11-15T00:44:34,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38979,DS-b52a2a7a-806c-4b77-aaee-5676bb37066d,DISK] 2024-11-15T00:44:34,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T00:44:34,586 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/WALs/fb2f6cb7846a,33521,1731631471305/fb2f6cb7846a%2C33521%2C1731631471305.1731631474474 2024-11-15T00:44:34,588 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38505:38505)] 2024-11-15T00:44:34,588 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:44:34,589 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:44:34,594 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,596 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,647 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T00:44:34,680 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:34,682 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:44:34,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T00:44:34,687 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:34,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:44:34,689 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T00:44:34,692 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:34,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:44:34,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T00:44:34,697 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:34,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:44:34,700 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,705 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,707 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,716 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,716 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,722 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T00:44:34,728 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:44:34,736 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:44:34,737 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59850687, jitterRate=-0.1081552654504776}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T00:44:34,744 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731631474613Initializing all the Stores at 1731631474617 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631474618 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631474620 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631474620Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631474621 (+1 ms)Cleaning up temporary data from old regions at 1731631474717 (+96 ms)Region opened successfully at 1731631474744 (+27 ms) 2024-11-15T00:44:34,746 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T00:44:34,794 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce2be35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fb2f6cb7846a/172.17.0.3:0 2024-11-15T00:44:34,828 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T00:44:34,843 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T00:44:34,843 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T00:44:34,847 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:44:34,849 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:44:34,855 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T00:44:34,855 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T00:44:34,884 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T00:44:34,897 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T00:44:34,910 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T00:44:34,913 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T00:44:34,916 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T00:44:34,929 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T00:44:34,932 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T00:44:34,937 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T00:44:34,950 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T00:44:34,955 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T00:44:34,963 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T00:44:34,991 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T00:44:35,005 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T00:44:35,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T00:44:35,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T00:44:35,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:35,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:35,018 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fb2f6cb7846a,33521,1731631471305, sessionid=0x1013bf352ec0000, setting cluster-up flag (Was=false) 2024-11-15T00:44:35,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:35,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:35,089 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T00:44:35,091 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fb2f6cb7846a,33521,1731631471305 2024-11-15T00:44:35,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:35,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:35,152 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T00:44:35,154 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fb2f6cb7846a,33521,1731631471305 2024-11-15T00:44:35,162 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T00:44:35,204 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(746): ClusterId : 78911300-2cd1-4362-aa5b-886d0f47db71 2024-11-15T00:44:35,207 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T00:44:35,218 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T00:44:35,218 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T00:44:35,228 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T00:44:35,229 DEBUG [RS:0;fb2f6cb7846a:34537 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c7ddf8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fb2f6cb7846a/172.17.0.3:0 2024-11-15T00:44:35,245 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:44:35,249 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fb2f6cb7846a:34537 2024-11-15T00:44:35,253 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T00:44:35,253 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T00:44:35,253 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T00:44:35,258 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T00:44:35,258 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(2659): reportForDuty to master=fb2f6cb7846a,33521,1731631471305 with port=34537, startcode=1731631472208 2024-11-15T00:44:35,269 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T00:44:35,273 DEBUG [RS:0;fb2f6cb7846a:34537 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T00:44:35,277 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fb2f6cb7846a,33521,1731631471305 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T00:44:35,292 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:44:35,292 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:44:35,292 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:44:35,293 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:44:35,293 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fb2f6cb7846a:0, corePoolSize=10, maxPoolSize=10 2024-11-15T00:44:35,293 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,293 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=2, maxPoolSize=2 2024-11-15T00:44:35,293 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,305 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631505305 2024-11-15T00:44:35,307 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T00:44:35,308 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T00:44:35,310 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T00:44:35,312 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T00:44:35,318 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T00:44:35,319 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T00:44:35,319 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T00:44:35,319 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T00:44:35,321 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,327 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:35,327 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T00:44:35,333 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T00:44:35,338 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T00:44:35,339 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51567, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T00:44:35,339 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T00:44:35,348 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:35,354 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T00:44:35,354 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T00:44:35,368 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.large.0-1731631475356,5,FailOnTimeoutGroup] 2024-11-15T00:44:35,372 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.small.0-1731631475368,5,FailOnTimeoutGroup] 2024-11-15T00:44:35,372 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,373 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T00:44:35,374 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,375 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,405 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-15T00:44:35,405 WARN [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-15T00:44:35,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741831_1007 (size=1321) 2024-11-15T00:44:35,427 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T00:44:35,428 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403 2024-11-15T00:44:35,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741832_1008 (size=32) 2024-11-15T00:44:35,459 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:44:35,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T00:44:35,472 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T00:44:35,472 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:35,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:44:35,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T00:44:35,482 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T00:44:35,483 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:35,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:44:35,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T00:44:35,494 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T00:44:35,494 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:35,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:44:35,497 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T00:44:35,513 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(2659): reportForDuty to master=fb2f6cb7846a,33521,1731631471305 with port=34537, startcode=1731631472208 2024-11-15T00:44:35,516 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:35,518 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T00:44:35,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:35,519 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] master.ServerManager(517): Registering regionserver=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:35,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:44:35,522 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T00:44:35,533 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740 2024-11-15T00:44:35,538 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403 2024-11-15T00:44:35,538 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42881 2024-11-15T00:44:35,539 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T00:44:35,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740 2024-11-15T00:44:35,549 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T00:44:35,549 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T00:44:35,551 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:44:35,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T00:44:35,556 DEBUG [RS:0;fb2f6cb7846a:34537 {}] zookeeper.ZKUtil(111): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:35,557 WARN [RS:0;fb2f6cb7846a:34537 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T00:44:35,557 INFO [RS:0;fb2f6cb7846a:34537 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T00:44:35,557 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/WALs/fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:35,560 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T00:44:35,566 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fb2f6cb7846a,34537,1731631472208] 2024-11-15T00:44:35,580 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:44:35,588 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62016064, jitterRate=-0.07588863372802734}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:44:35,593 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731631475459Initializing all the Stores at 1731631475465 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631475465Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631475468 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631475468Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631475468Cleaning up temporary data from old regions at 1731631475549 (+81 ms)Region opened successfully at 1731631475593 (+44 ms) 2024-11-15T00:44:35,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T00:44:35,594 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T00:44:35,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T00:44:35,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T00:44:35,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T00:44:35,601 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T00:44:35,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731631475594Disabling compacts and flushes for region at 1731631475594Disabling writes for close at 1731631475594Writing region close event to WAL at 1731631475601 (+7 ms)Closed at 1731631475601 2024-11-15T00:44:35,609 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T00:44:35,609 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T00:44:35,616 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T00:44:35,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T00:44:35,631 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T00:44:35,642 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T00:44:35,648 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T00:44:35,659 INFO [RS:0;fb2f6cb7846a:34537 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T00:44:35,659 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,662 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T00:44:35,685 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T00:44:35,687 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,687 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,687 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,688 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,688 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,688 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,688 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fb2f6cb7846a:0, corePoolSize=2, maxPoolSize=2 2024-11-15T00:44:35,688 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,689 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,689 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,689 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,689 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,689 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:44:35,689 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fb2f6cb7846a:0, corePoolSize=3, maxPoolSize=3 2024-11-15T00:44:35,690 DEBUG [RS:0;fb2f6cb7846a:34537 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0, corePoolSize=3, maxPoolSize=3 2024-11-15T00:44:35,693 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,694 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,694 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,694 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,694 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,694 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,34537,1731631472208-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T00:44:35,726 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T00:44:35,728 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,34537,1731631472208-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,729 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,729 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.Replication(171): fb2f6cb7846a,34537,1731631472208 started 2024-11-15T00:44:35,754 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:35,755 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1482): Serving as fb2f6cb7846a,34537,1731631472208, RpcServer on fb2f6cb7846a/172.17.0.3:34537, sessionid=0x1013bf352ec0001 2024-11-15T00:44:35,756 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T00:44:35,757 DEBUG [RS:0;fb2f6cb7846a:34537 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:35,757 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fb2f6cb7846a,34537,1731631472208' 2024-11-15T00:44:35,757 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T00:44:35,758 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T00:44:35,764 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T00:44:35,765 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T00:44:35,765 DEBUG [RS:0;fb2f6cb7846a:34537 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:35,765 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fb2f6cb7846a,34537,1731631472208' 2024-11-15T00:44:35,765 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T00:44:35,768 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T00:44:35,775 DEBUG [RS:0;fb2f6cb7846a:34537 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T00:44:35,775 INFO [RS:0;fb2f6cb7846a:34537 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T00:44:35,776 INFO [RS:0;fb2f6cb7846a:34537 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T00:44:35,796 WARN [fb2f6cb7846a:33521 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-15T00:44:35,883 INFO [RS:0;fb2f6cb7846a:34537 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T00:44:35,889 INFO [RS:0;fb2f6cb7846a:34537 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fb2f6cb7846a%2C34537%2C1731631472208, suffix=, logDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/WALs/fb2f6cb7846a,34537,1731631472208, archiveDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/oldWALs, maxLogs=32 2024-11-15T00:44:35,913 DEBUG [RS:0;fb2f6cb7846a:34537 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/WALs/fb2f6cb7846a,34537,1731631472208/fb2f6cb7846a%2C34537%2C1731631472208.1731631475893, exclude list is [], retry=0 2024-11-15T00:44:35,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38979,DS-b52a2a7a-806c-4b77-aaee-5676bb37066d,DISK] 2024-11-15T00:44:35,927 INFO [RS:0;fb2f6cb7846a:34537 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/WALs/fb2f6cb7846a,34537,1731631472208/fb2f6cb7846a%2C34537%2C1731631472208.1731631475893 2024-11-15T00:44:35,928 DEBUG [RS:0;fb2f6cb7846a:34537 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38505:38505)] 2024-11-15T00:44:36,048 DEBUG [fb2f6cb7846a:33521 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T00:44:36,063 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:36,073 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,34537,1731631472208, state=OPENING 2024-11-15T00:44:36,089 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T00:44:36,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:36,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:44:36,101 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:44:36,101 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:44:36,103 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T00:44:36,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:44:36,286 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T00:44:36,291 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40527, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T00:44:36,312 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T00:44:36,313 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T00:44:36,315 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T00:44:36,323 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fb2f6cb7846a%2C34537%2C1731631472208.meta, suffix=.meta, logDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/WALs/fb2f6cb7846a,34537,1731631472208, archiveDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/oldWALs, maxLogs=32 2024-11-15T00:44:36,348 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/WALs/fb2f6cb7846a,34537,1731631472208/fb2f6cb7846a%2C34537%2C1731631472208.meta.1731631476326.meta, exclude list is [], retry=0 2024-11-15T00:44:36,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38979,DS-b52a2a7a-806c-4b77-aaee-5676bb37066d,DISK] 2024-11-15T00:44:36,361 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/WALs/fb2f6cb7846a,34537,1731631472208/fb2f6cb7846a%2C34537%2C1731631472208.meta.1731631476326.meta 2024-11-15T00:44:36,362 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38505:38505)] 2024-11-15T00:44:36,362 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:44:36,365 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T00:44:36,369 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T00:44:36,375 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T00:44:36,382 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T00:44:36,383 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:44:36,383 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T00:44:36,383 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T00:44:36,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T00:44:36,399 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T00:44:36,400 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:36,401 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:44:36,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T00:44:36,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T00:44:36,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:36,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:44:36,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T00:44:36,416 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T00:44:36,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:36,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:44:36,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T00:44:36,421 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T00:44:36,421 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:36,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:44:36,428 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T00:44:36,432 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740 2024-11-15T00:44:36,437 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740 2024-11-15T00:44:36,447 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T00:44:36,447 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T00:44:36,449 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:44:36,453 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T00:44:36,458 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59003504, jitterRate=-0.12077927589416504}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:44:36,458 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T00:44:36,461 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731631476384Writing region info on filesystem at 1731631476384Initializing all the Stores at 1731631476389 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631476389Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631476392 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631476392Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631476392Cleaning up temporary data from old regions at 1731631476447 (+55 ms)Running coprocessor post-open hooks at 1731631476459 (+12 ms)Region opened successfully at 1731631476460 (+1 ms) 2024-11-15T00:44:36,475 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731631476276 2024-11-15T00:44:36,502 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T00:44:36,503 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T00:44:36,510 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:36,517 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,34537,1731631472208, state=OPEN 2024-11-15T00:44:36,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:44:36,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:44:36,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:44:36,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:44:36,615 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:36,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T00:44:36,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fb2f6cb7846a,34537,1731631472208 in 510 msec 2024-11-15T00:44:36,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T00:44:36,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0070 sec 2024-11-15T00:44:36,638 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T00:44:36,639 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T00:44:36,663 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:36,665 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:36,687 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:36,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55973, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:36,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5220 sec 2024-11-15T00:44:36,724 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731631476723, completionTime=-1 2024-11-15T00:44:36,727 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T00:44:36,728 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:44:36,768 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:44:36,768 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631536768 2024-11-15T00:44:36,769 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631596769 2024-11-15T00:44:36,769 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 41 msec 2024-11-15T00:44:36,771 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,33521,1731631471305-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:36,772 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,33521,1731631471305-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:36,772 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,33521,1731631471305-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:36,774 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fb2f6cb7846a:33521, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:36,778 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:36,784 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:36,786 DEBUG [master/fb2f6cb7846a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T00:44:36,818 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.453sec 2024-11-15T00:44:36,820 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T00:44:36,821 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T00:44:36,823 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T00:44:36,823 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T00:44:36,824 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T00:44:36,825 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,33521,1731631471305-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T00:44:36,826 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,33521,1731631471305-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T00:44:36,836 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T00:44:36,837 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T00:44:36,838 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,33521,1731631471305-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:44:36,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:36,919 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T00:44:36,919 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T00:44:36,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:36,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:36,935 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301f7ee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:36,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:36,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:36,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:36,951 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:36,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:36,955 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:36,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:37,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fb2f6cb7846a,33521,1731631471305 2024-11-15T00:44:37,036 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=534, ProcessCount=11, AvailableMemoryMB=2940 2024-11-15T00:44:37,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:44:37,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,33521,1731631471305 2024-11-15T00:44:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b1db67 2024-11-15T00:44:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:44:37,083 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:44:37,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:44:37,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:44:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:44:37,104 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:44:37,107 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:37,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T00:44:37,110 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:44:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:44:37,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741835_1011 (size=1061) 2024-11-15T00:44:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:44:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:44:37,536 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403 2024-11-15T00:44:37,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741836_1012 (size=53) 2024-11-15T00:44:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:44:37,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:44:37,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c6201d6aebcecefaa2f781f0e095054e, disabling compactions & flushes 2024-11-15T00:44:37,977 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:37,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:37,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. after waiting 0 ms 2024-11-15T00:44:37,978 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:37,978 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:37,978 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c6201d6aebcecefaa2f781f0e095054e: Waiting for close lock at 1731631477977Disabling compacts and flushes for region at 1731631477977Disabling writes for close at 1731631477977Writing region close event to WAL at 1731631477978 (+1 ms)Closed at 1731631477978 2024-11-15T00:44:37,981 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:44:37,992 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631477982"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631477982"}]},"ts":"1731631477982"} 2024-11-15T00:44:38,015 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:44:38,019 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:44:38,023 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631478019"}]},"ts":"1731631478019"} 2024-11-15T00:44:38,039 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:44:38,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6201d6aebcecefaa2f781f0e095054e, ASSIGN}] 2024-11-15T00:44:38,048 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6201d6aebcecefaa2f781f0e095054e, ASSIGN 2024-11-15T00:44:38,051 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6201d6aebcecefaa2f781f0e095054e, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,34537,1731631472208; forceNewPlan=false, retain=false 2024-11-15T00:44:38,205 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c6201d6aebcecefaa2f781f0e095054e, regionState=OPENING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:38,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6201d6aebcecefaa2f781f0e095054e, ASSIGN because future has completed 2024-11-15T00:44:38,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:44:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:44:38,440 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:38,441 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:44:38,441 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,442 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:44:38,442 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,442 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,450 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,456 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:44:38,457 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6201d6aebcecefaa2f781f0e095054e columnFamilyName A 2024-11-15T00:44:38,457 DEBUG [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:38,459 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] regionserver.HStore(327): Store=c6201d6aebcecefaa2f781f0e095054e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:44:38,459 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,462 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:44:38,463 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6201d6aebcecefaa2f781f0e095054e columnFamilyName B 2024-11-15T00:44:38,463 DEBUG [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:38,464 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] regionserver.HStore(327): Store=c6201d6aebcecefaa2f781f0e095054e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:44:38,464 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,467 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:44:38,468 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6201d6aebcecefaa2f781f0e095054e columnFamilyName C 2024-11-15T00:44:38,468 DEBUG [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:44:38,469 INFO [StoreOpener-c6201d6aebcecefaa2f781f0e095054e-1 {}] regionserver.HStore(327): Store=c6201d6aebcecefaa2f781f0e095054e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:44:38,470 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:38,470 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,472 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,473 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,476 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,476 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,477 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:44:38,481 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,490 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:44:38,499 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c6201d6aebcecefaa2f781f0e095054e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62594382, jitterRate=-0.06727102398872375}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:44:38,499 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:38,501 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c6201d6aebcecefaa2f781f0e095054e: Running coprocessor pre-open hook at 1731631478442Writing region info on filesystem at 1731631478442Initializing all the Stores at 1731631478448 (+6 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631478449 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631478450 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631478450Cleaning up temporary data from old regions at 1731631478476 (+26 ms)Running coprocessor post-open hooks at 1731631478499 (+23 ms)Region opened successfully at 1731631478501 (+2 ms) 2024-11-15T00:44:38,509 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., pid=6, masterSystemTime=1731631478408 2024-11-15T00:44:38,516 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c6201d6aebcecefaa2f781f0e095054e, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:38,520 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:38,520 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:38,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:44:38,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T00:44:38,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 in 288 msec 2024-11-15T00:44:38,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T00:44:38,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6201d6aebcecefaa2f781f0e095054e, ASSIGN in 493 msec 2024-11-15T00:44:38,544 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:44:38,545 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631478545"}]},"ts":"1731631478545"} 2024-11-15T00:44:38,549 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:44:38,551 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:44:38,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.4550 sec 2024-11-15T00:44:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:44:39,273 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:44:39,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:39,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:39,293 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb5e284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:39,297 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17282950, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,306 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:39,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:39,336 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:44:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,33521,1731631471305 2024-11-15T00:44:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@24720917 2024-11-15T00:44:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:44:39,349 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:44:39,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:39,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:39,360 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:39,364 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:39,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:39,368 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:39,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:39,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:39,378 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:39,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:39,387 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b37f010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:39,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:39,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:39,408 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:39,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:39,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:39,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a8011b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:39,414 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:39,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c7aae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:39,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:39,422 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4a2a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:39,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:39,425 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738bed33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:39,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:39,430 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50131031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:39,434 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:39,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359c1e82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:39,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:39,445 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48230300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:39,450 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ba8772, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:39,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:39,471 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272ed7e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:39,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:39,476 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477e84a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:44:39,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:44:39,490 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:44:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:44:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:44:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441ebf29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:44:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:44:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:39,494 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:44:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b6ade8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:44:39,509 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:39,510 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:39,516 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:39,517 DEBUG [Thread-149 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:39,518 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:39,518 DEBUG [Thread-149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:39,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:39,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:44:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:44:39,528 DEBUG [Thread-149 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:44:39,530 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:39,534 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:44:39,545 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:39,549 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:39,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:39,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:44:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:44:39,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:44:39,573 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:39,577 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T00:44:39,579 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:44:39,580 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:44:39,583 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:44:39,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:44:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:44:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:39,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:39,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:39,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:39,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:44:39,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:44:39,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:44:39,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:44:39,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:44:39,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:44:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:39,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T00:44:39,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:39,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:39,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:39,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:39,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:39,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:44:39,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:39,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:39,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:39,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:39,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:39,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:39,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631479795, max timestamp=1731631479832Num uniques 859; 2024-11-15T00:44:39,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:44:39,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631479795, max timestamp=1731631479832Num uniques 859; 2024-11-15T00:44:39,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:44:39,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631479795, max timestamp=1731631479832Num uniques 859; 2024-11-15T00:44:39,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:44:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631539811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:39,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:39,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:39,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:39,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/d16343887e6244368ee22e7699e24533 is 50, key is test_row_0/A:col10/1731631479634/Put/seqid=0 2024-11-15T00:44:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:44:39,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741837_1013 (size=12001) 2024-11-15T00:44:39,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:39,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:39,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:39,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:39,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:39,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631539986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631540000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631540025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631540036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631540036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631540154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631540154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631540162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631540162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631540162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:40,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:40,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:44:40,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/d16343887e6244368ee22e7699e24533 2024-11-15T00:44:40,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:40,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:40,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:40,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:40,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631540372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631540376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631540381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631540382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631540390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:40,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/f98545b0aff14f068e2b329b718d482b is 50, key is test_row_0/B:col10/1731631479634/Put/seqid=0 2024-11-15T00:44:40,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741838_1014 (size=12001) 2024-11-15T00:44:40,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/f98545b0aff14f068e2b329b718d482b 2024-11-15T00:44:40,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:40,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:40,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:40,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:40,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631540702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631540702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631540702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/22737661330f4dad895db61a5a5ba53c is 50, key is test_row_0/C:col10/1731631479634/Put/seqid=0 2024-11-15T00:44:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:44:40,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631540714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631540736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:40,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741839_1015 (size=12001) 2024-11-15T00:44:40,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:40,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/22737661330f4dad895db61a5a5ba53c 2024-11-15T00:44:41,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:41,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:41,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/d16343887e6244368ee22e7699e24533 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d16343887e6244368ee22e7699e24533 2024-11-15T00:44:41,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d16343887e6244368ee22e7699e24533, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T00:44:41,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/f98545b0aff14f068e2b329b718d482b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/f98545b0aff14f068e2b329b718d482b 2024-11-15T00:44:41,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:41,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631541209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:41,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:41,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631541220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:41,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631541222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:41,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/f98545b0aff14f068e2b329b718d482b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T00:44:41,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/22737661330f4dad895db61a5a5ba53c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/22737661330f4dad895db61a5a5ba53c 2024-11-15T00:44:41,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/22737661330f4dad895db61a5a5ba53c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T00:44:41,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.13 KB/120965 for c6201d6aebcecefaa2f781f0e095054e in 1626ms, sequenceid=13, compaction requested=false 2024-11-15T00:44:41,297 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T00:44:41,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:41,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-11-15T00:44:41,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:41,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:41,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:41,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:41,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:41,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:41,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:41,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:41,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/3293fdbf557b4be2a2c978b08b14cd9b is 50, key is test_row_0/A:col10/1731631481293/Put/seqid=0 2024-11-15T00:44:41,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741840_1016 (size=16681) 2024-11-15T00:44:41,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/3293fdbf557b4be2a2c978b08b14cd9b 2024-11-15T00:44:41,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/7af1e4b1f66f4a20b2ad9b012168e65f is 50, key is test_row_0/B:col10/1731631481293/Put/seqid=0 2024-11-15T00:44:41,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741841_1017 (size=12001) 2024-11-15T00:44:41,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/7af1e4b1f66f4a20b2ad9b012168e65f 2024-11-15T00:44:41,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:41,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:41,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/8c31cfcb8ce6495c895d570296d689e3 is 50, key is test_row_0/C:col10/1731631481293/Put/seqid=0 2024-11-15T00:44:41,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741842_1018 (size=12001) 2024-11-15T00:44:41,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631541595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:41,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631541595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:41,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:44:41,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631541746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631541747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:41,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,902 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:44:41,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631541960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:41,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631541963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:41,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:41,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:42,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/8c31cfcb8ce6495c895d570296d689e3 2024-11-15T00:44:42,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/3293fdbf557b4be2a2c978b08b14cd9b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3293fdbf557b4be2a2c978b08b14cd9b 2024-11-15T00:44:42,044 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T00:44:42,046 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T00:44:42,047 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T00:44:42,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3293fdbf557b4be2a2c978b08b14cd9b, entries=250, sequenceid=49, filesize=16.3 K 2024-11-15T00:44:42,049 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T00:44:42,049 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T00:44:42,049 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T00:44:42,049 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T00:44:42,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/7af1e4b1f66f4a20b2ad9b012168e65f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/7af1e4b1f66f4a20b2ad9b012168e65f 2024-11-15T00:44:42,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T00:44:42,052 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T00:44:42,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/7af1e4b1f66f4a20b2ad9b012168e65f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T00:44:42,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/8c31cfcb8ce6495c895d570296d689e3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8c31cfcb8ce6495c895d570296d689e3 2024-11-15T00:44:42,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8c31cfcb8ce6495c895d570296d689e3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T00:44:42,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~374.26 KB/383240, currentSize=53.67 KB/54960 for c6201d6aebcecefaa2f781f0e095054e in 781ms, sequenceid=49, compaction requested=false 2024-11-15T00:44:42,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:42,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:44:42,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:42,154 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T00:44:42,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:42,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:42,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:42,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:42,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:42,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/1e2462a5743a4de0be815014eab1ab95 is 50, key is test_row_0/A:col10/1731631481503/Put/seqid=0 2024-11-15T00:44:42,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741843_1019 (size=12001) 2024-11-15T00:44:42,260 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/1e2462a5743a4de0be815014eab1ab95 2024-11-15T00:44:42,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/5f8979eae2f0478a814563f826a9d5a1 is 50, key is test_row_0/B:col10/1731631481503/Put/seqid=0 2024-11-15T00:44:42,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741844_1020 (size=12001) 2024-11-15T00:44:42,363 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/5f8979eae2f0478a814563f826a9d5a1 2024-11-15T00:44:42,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631482317, max timestamp=1731631482372Num uniques 859; 2024-11-15T00:44:42,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:44:42,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631482317, max timestamp=1731631482372Num uniques 859; 2024-11-15T00:44:42,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:44:42,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/a007d155108c4ab1b0aba4a7238a744a is 50, key is test_row_0/C:col10/1731631481503/Put/seqid=0 2024-11-15T00:44:42,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631482317, max timestamp=1731631482372Num uniques 859; 2024-11-15T00:44:42,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:44:42,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741845_1021 (size=12001) 2024-11-15T00:44:42,462 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/a007d155108c4ab1b0aba4a7238a744a 2024-11-15T00:44:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/1e2462a5743a4de0be815014eab1ab95 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/1e2462a5743a4de0be815014eab1ab95 2024-11-15T00:44:42,517 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/1e2462a5743a4de0be815014eab1ab95, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T00:44:42,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/5f8979eae2f0478a814563f826a9d5a1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f8979eae2f0478a814563f826a9d5a1 2024-11-15T00:44:42,565 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f8979eae2f0478a814563f826a9d5a1, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T00:44:42,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/a007d155108c4ab1b0aba4a7238a744a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a007d155108c4ab1b0aba4a7238a744a 2024-11-15T00:44:42,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631542516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631542537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631542539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,587 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a007d155108c4ab1b0aba4a7238a744a, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T00:44:42,590 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.02 KB/139285 for c6201d6aebcecefaa2f781f0e095054e in 436ms, sequenceid=60, compaction requested=true 2024-11-15T00:44:42,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:42,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:42,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T00:44:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T00:44:42,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=142.73 KB heapSize=404.38 KB 2024-11-15T00:44:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:42,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T00:44:42,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0250 sec 2024-11-15T00:44:42,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731631482505, max timestamp=1731631482596Num uniques 1118; 2024-11-15T00:44:42,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731631482505, max timestamp=1731631482596Num uniques 1118; 2024-11-15T00:44:42,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-15T00:44:42,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-15T00:44:42,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:42,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:42,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:42,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:42,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:42,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T00:44:42,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.0600 sec 2024-11-15T00:44:42,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/60a4c9d2ad0b464f9a73859e2dab6921 is 50, key is test_row_0/A:col10/1731631482589/Put/seqid=0 2024-11-15T00:44:42,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741846_1022 (size=14341) 2024-11-15T00:44:42,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631542764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631542764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631542771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631542771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631542785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631542889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631542902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631542902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631542903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:42,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631542921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/60a4c9d2ad0b464f9a73859e2dab6921 2024-11-15T00:44:43,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631543113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:43,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631543122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631543122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631543122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631543141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/6723ab6b53374955b5b21962d10efd87 is 50, key is test_row_0/B:col10/1731631482589/Put/seqid=0 2024-11-15T00:44:43,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741847_1023 (size=12001) 2024-11-15T00:44:43,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/6723ab6b53374955b5b21962d10efd87 2024-11-15T00:44:43,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/3ec65758c0aa4e4cba47f1257b67ab38 is 50, key is test_row_0/C:col10/1731631482589/Put/seqid=0 2024-11-15T00:44:43,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741848_1024 (size=12001) 2024-11-15T00:44:43,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631543440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:43,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631543440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631543441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631543454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631543455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:43,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/3ec65758c0aa4e4cba47f1257b67ab38 2024-11-15T00:44:43,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/60a4c9d2ad0b464f9a73859e2dab6921 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/60a4c9d2ad0b464f9a73859e2dab6921 2024-11-15T00:44:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:44:43,766 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:44:43,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:44:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T00:44:43,775 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:44:43,776 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:44:43,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:44:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T00:44:43,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/60a4c9d2ad0b464f9a73859e2dab6921, entries=200, sequenceid=98, filesize=14.0 K 2024-11-15T00:44:43,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/6723ab6b53374955b5b21962d10efd87 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/6723ab6b53374955b5b21962d10efd87 2024-11-15T00:44:43,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/6723ab6b53374955b5b21962d10efd87, entries=150, sequenceid=98, filesize=11.7 K 2024-11-15T00:44:43,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/3ec65758c0aa4e4cba47f1257b67ab38 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/3ec65758c0aa4e4cba47f1257b67ab38 2024-11-15T00:44:43,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/3ec65758c0aa4e4cba47f1257b67ab38, entries=150, sequenceid=98, filesize=11.7 K 2024-11-15T00:44:43,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.46 KB/66005, heapSize ~177.84 KB/182104, currentSize=114.05 KB/116790 for c6201d6aebcecefaa2f781f0e095054e in 1250ms, sequenceid=98, compaction requested=true 2024-11-15T00:44:43,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:43,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:44:43,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:43,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:44:43,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:43,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:44:43,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:44:43,871 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:43,881 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T00:44:43,901 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:43,903 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/B is initiating minor compaction (all files) 2024-11-15T00:44:43,904 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/B in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:43,905 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/f98545b0aff14f068e2b329b718d482b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/7af1e4b1f66f4a20b2ad9b012168e65f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f8979eae2f0478a814563f826a9d5a1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/6723ab6b53374955b5b21962d10efd87] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=46.9 K 2024-11-15T00:44:43,909 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:43,910 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/A is initiating minor compaction (all files) 2024-11-15T00:44:43,910 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/A in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:43,910 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d16343887e6244368ee22e7699e24533, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3293fdbf557b4be2a2c978b08b14cd9b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/1e2462a5743a4de0be815014eab1ab95, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/60a4c9d2ad0b464f9a73859e2dab6921] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=53.7 K 2024-11-15T00:44:43,912 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f98545b0aff14f068e2b329b718d482b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731631479623 2024-11-15T00:44:43,918 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting d16343887e6244368ee22e7699e24533, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731631479623 2024-11-15T00:44:43,919 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af1e4b1f66f4a20b2ad9b012168e65f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731631479972 2024-11-15T00:44:43,922 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3293fdbf557b4be2a2c978b08b14cd9b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731631479972 2024-11-15T00:44:43,922 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f8979eae2f0478a814563f826a9d5a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731631481503 2024-11-15T00:44:43,925 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e2462a5743a4de0be815014eab1ab95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731631481503 2024-11-15T00:44:43,929 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 6723ab6b53374955b5b21962d10efd87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731631482536 2024-11-15T00:44:43,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T00:44:43,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:43,932 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:44:43,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:43,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:43,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:43,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:43,933 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60a4c9d2ad0b464f9a73859e2dab6921, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731631482505 2024-11-15T00:44:44,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:44,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/bd7d7f1238b04284bccc5d7b1468543b is 50, key is test_row_0/A:col10/1731631482760/Put/seqid=0 2024-11-15T00:44:44,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741849_1025 (size=12001) 2024-11-15T00:44:44,041 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/bd7d7f1238b04284bccc5d7b1468543b 2024-11-15T00:44:44,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631544030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T00:44:44,097 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#A#compaction#14 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631544049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,098 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/7011b47e83ab46f68d472cb286f6ca7b is 50, key is test_row_0/A:col10/1731631482589/Put/seqid=0 2024-11-15T00:44:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,101 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#B#compaction#13 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:44,102 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/31ea9b072cc846599adba8a7f753e68d is 50, key is test_row_0/B:col10/1731631482589/Put/seqid=0 2024-11-15T00:44:44,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/2d9a385a822b4a5ebf9bf7e8764821ec is 50, key is test_row_0/B:col10/1731631482760/Put/seqid=0 2024-11-15T00:44:44,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631544056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631544061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631544070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741851_1027 (size=12139) 2024-11-15T00:44:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741850_1026 (size=12139) 2024-11-15T00:44:44,180 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/31ea9b072cc846599adba8a7f753e68d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/31ea9b072cc846599adba8a7f753e68d 2024-11-15T00:44:44,192 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/7011b47e83ab46f68d472cb286f6ca7b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/7011b47e83ab46f68d472cb286f6ca7b 2024-11-15T00:44:44,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631544184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631544209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,229 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/B of c6201d6aebcecefaa2f781f0e095054e into 31ea9b072cc846599adba8a7f753e68d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:44,229 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:44,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741852_1028 (size=12001) 2024-11-15T00:44:44,230 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/A of c6201d6aebcecefaa2f781f0e095054e into 7011b47e83ab46f68d472cb286f6ca7b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:44,231 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:44,232 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/2d9a385a822b4a5ebf9bf7e8764821ec 2024-11-15T00:44:44,234 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/A, priority=12, startTime=1731631483855; duration=0sec 2024-11-15T00:44:44,236 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/B, priority=12, startTime=1731631483863; duration=0sec 2024-11-15T00:44:44,237 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:44,238 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:B 2024-11-15T00:44:44,238 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:44,242 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:44,243 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:A 2024-11-15T00:44:44,253 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:44,253 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/C is initiating minor compaction (all files) 2024-11-15T00:44:44,253 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/C in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:44,253 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/22737661330f4dad895db61a5a5ba53c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8c31cfcb8ce6495c895d570296d689e3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a007d155108c4ab1b0aba4a7238a744a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/3ec65758c0aa4e4cba47f1257b67ab38] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=46.9 K 2024-11-15T00:44:44,255 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 22737661330f4dad895db61a5a5ba53c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731631479623 2024-11-15T00:44:44,256 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c31cfcb8ce6495c895d570296d689e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731631479972 2024-11-15T00:44:44,257 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a007d155108c4ab1b0aba4a7238a744a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731631481503 2024-11-15T00:44:44,259 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec65758c0aa4e4cba47f1257b67ab38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731631482536 2024-11-15T00:44:44,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/55ab3e4198d64897b15f758037522f99 is 50, key is test_row_0/C:col10/1731631482760/Put/seqid=0 2024-11-15T00:44:44,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631544256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631544256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631544259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,319 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#C#compaction#17 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:44,320 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/6c3bdbd0562c467eb4c097dcf5087a83 is 50, key is test_row_0/C:col10/1731631482589/Put/seqid=0 2024-11-15T00:44:44,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741853_1029 (size=12001) 2024-11-15T00:44:44,325 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/55ab3e4198d64897b15f758037522f99 2024-11-15T00:44:44,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/bd7d7f1238b04284bccc5d7b1468543b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/bd7d7f1238b04284bccc5d7b1468543b 2024-11-15T00:44:44,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741854_1030 (size=12139) 2024-11-15T00:44:44,363 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/bd7d7f1238b04284bccc5d7b1468543b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T00:44:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/2d9a385a822b4a5ebf9bf7e8764821ec as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/2d9a385a822b4a5ebf9bf7e8764821ec 2024-11-15T00:44:44,368 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/6c3bdbd0562c467eb4c097dcf5087a83 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/6c3bdbd0562c467eb4c097dcf5087a83 2024-11-15T00:44:44,382 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/2d9a385a822b4a5ebf9bf7e8764821ec, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T00:44:44,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/55ab3e4198d64897b15f758037522f99 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/55ab3e4198d64897b15f758037522f99 2024-11-15T00:44:44,394 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/C of c6201d6aebcecefaa2f781f0e095054e into 6c3bdbd0562c467eb4c097dcf5087a83(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:44,395 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:44,395 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/C, priority=12, startTime=1731631483864; duration=0sec 2024-11-15T00:44:44,395 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:44,395 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:C 2024-11-15T00:44:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T00:44:44,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631544400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,416 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/55ab3e4198d64897b15f758037522f99, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T00:44:44,437 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c6201d6aebcecefaa2f781f0e095054e in 505ms, sequenceid=118, compaction requested=false 2024-11-15T00:44:44,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:44,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:44,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T00:44:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T00:44:44,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T00:44:44,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 664 msec 2024-11-15T00:44:44,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 678 msec 2024-11-15T00:44:44,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:44:44,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:44,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:44,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:44,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:44,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:44,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:44,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/c069c0b8a2b0481e85f667cdef2341e1 is 50, key is test_row_0/A:col10/1731631484452/Put/seqid=0 2024-11-15T00:44:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741855_1031 (size=12097) 2024-11-15T00:44:44,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/c069c0b8a2b0481e85f667cdef2341e1 2024-11-15T00:44:44,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/3fdb1ac96d604ccf91a7ef54881fa753 is 50, key is test_row_0/B:col10/1731631484452/Put/seqid=0 2024-11-15T00:44:44,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631544566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631544580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631544590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631544602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741856_1032 (size=9707) 2024-11-15T00:44:44,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631544699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631544700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631544712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631544720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631544720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T00:44:44,910 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:44:44,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:44:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T00:44:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:44:44,917 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:44:44,919 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:44:44,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:44:44,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631544910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631544919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631544940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:44,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:44,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631544951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:44:45,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/3fdb1ac96d604ccf91a7ef54881fa753 2024-11-15T00:44:45,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/fe476462f8084655a461d30c4f1a434f is 50, key is test_row_0/C:col10/1731631484452/Put/seqid=0 2024-11-15T00:44:45,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T00:44:45,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:45,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:45,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:45,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:45,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:45,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741857_1033 (size=9707) 2024-11-15T00:44:45,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/fe476462f8084655a461d30c4f1a434f 2024-11-15T00:44:45,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/c069c0b8a2b0481e85f667cdef2341e1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/c069c0b8a2b0481e85f667cdef2341e1 2024-11-15T00:44:45,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/c069c0b8a2b0481e85f667cdef2341e1, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T00:44:45,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/3fdb1ac96d604ccf91a7ef54881fa753 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fdb1ac96d604ccf91a7ef54881fa753 2024-11-15T00:44:45,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fdb1ac96d604ccf91a7ef54881fa753, entries=100, sequenceid=134, filesize=9.5 K 2024-11-15T00:44:45,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/fe476462f8084655a461d30c4f1a434f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/fe476462f8084655a461d30c4f1a434f 2024-11-15T00:44:45,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/fe476462f8084655a461d30c4f1a434f, entries=100, sequenceid=134, filesize=9.5 K 2024-11-15T00:44:45,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c6201d6aebcecefaa2f781f0e095054e in 742ms, sequenceid=134, compaction requested=true 2024-11-15T00:44:45,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:45,198 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:44:45,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:44:45,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:45,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:44:45,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:45,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:44:45,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:44:45,201 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:44:45,202 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/A is initiating minor compaction (all files) 2024-11-15T00:44:45,202 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/A in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:45,202 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:44:45,202 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/7011b47e83ab46f68d472cb286f6ca7b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/bd7d7f1238b04284bccc5d7b1468543b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/c069c0b8a2b0481e85f667cdef2341e1] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=35.4 K 2024-11-15T00:44:45,204 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 7011b47e83ab46f68d472cb286f6ca7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731631482536 2024-11-15T00:44:45,205 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting bd7d7f1238b04284bccc5d7b1468543b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731631482760 2024-11-15T00:44:45,207 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33847 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:44:45,207 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/B is initiating minor compaction (all files) 2024-11-15T00:44:45,207 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/B in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:45,207 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/31ea9b072cc846599adba8a7f753e68d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/2d9a385a822b4a5ebf9bf7e8764821ec, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fdb1ac96d604ccf91a7ef54881fa753] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=33.1 K 2024-11-15T00:44:45,208 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c069c0b8a2b0481e85f667cdef2341e1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631484048 2024-11-15T00:44:45,209 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31ea9b072cc846599adba8a7f753e68d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731631482536 2024-11-15T00:44:45,217 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d9a385a822b4a5ebf9bf7e8764821ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731631482760 2024-11-15T00:44:45,219 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fdb1ac96d604ccf91a7ef54881fa753, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631484048 2024-11-15T00:44:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:44:45,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T00:44:45,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:45,238 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:44:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:45,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:45,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:45,259 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#B#compaction#21 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:45,260 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/9f93bdae69434aa1a670bd57c18c40ad is 50, key is test_row_0/B:col10/1731631484452/Put/seqid=0 2024-11-15T00:44:45,280 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#A#compaction#23 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:45,281 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/8e2bef57724e4539835cd815b8fbe66b is 50, key is test_row_0/A:col10/1731631484452/Put/seqid=0 2024-11-15T00:44:45,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/ca930d7bd47746eca9fbaf9275c1ce39 is 50, key is test_row_0/A:col10/1731631484557/Put/seqid=0 2024-11-15T00:44:45,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631545287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631545303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631545314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631545316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631545317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741858_1034 (size=12291) 2024-11-15T00:44:45,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741859_1035 (size=12291) 2024-11-15T00:44:45,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631545420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741860_1036 (size=14541) 2024-11-15T00:44:45,455 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/8e2bef57724e4539835cd815b8fbe66b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/8e2bef57724e4539835cd815b8fbe66b 2024-11-15T00:44:45,456 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/ca930d7bd47746eca9fbaf9275c1ce39 2024-11-15T00:44:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631545450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631545451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631545460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631545460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,484 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/A of c6201d6aebcecefaa2f781f0e095054e into 8e2bef57724e4539835cd815b8fbe66b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:45,484 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:45,485 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/A, priority=13, startTime=1731631485198; duration=0sec 2024-11-15T00:44:45,485 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:45,485 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:A 2024-11-15T00:44:45,485 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:44:45,487 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33847 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:44:45,488 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/C is initiating minor compaction (all files) 2024-11-15T00:44:45,488 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/C in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:45,488 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/6c3bdbd0562c467eb4c097dcf5087a83, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/55ab3e4198d64897b15f758037522f99, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/fe476462f8084655a461d30c4f1a434f] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=33.1 K 2024-11-15T00:44:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/1789c40aa2ef4594a6c909099dd3e08c is 50, key is test_row_0/B:col10/1731631484557/Put/seqid=0 2024-11-15T00:44:45,498 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c3bdbd0562c467eb4c097dcf5087a83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731631482536 2024-11-15T00:44:45,500 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 55ab3e4198d64897b15f758037522f99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731631482760 2024-11-15T00:44:45,501 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting fe476462f8084655a461d30c4f1a434f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631484048 2024-11-15T00:44:45,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741861_1037 (size=12151) 2024-11-15T00:44:45,534 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#C#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:45,534 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/303b7fdd185946bba336e01270efecc6 is 50, key is test_row_0/C:col10/1731631484452/Put/seqid=0 2024-11-15T00:44:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:44:45,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741862_1038 (size=12291) 2024-11-15T00:44:45,619 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/303b7fdd185946bba336e01270efecc6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/303b7fdd185946bba336e01270efecc6 2024-11-15T00:44:45,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631545640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631545672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,676 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/C of c6201d6aebcecefaa2f781f0e095054e into 303b7fdd185946bba336e01270efecc6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:45,676 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:45,676 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/C, priority=13, startTime=1731631485201; duration=0sec 2024-11-15T00:44:45,676 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:45,676 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:C 2024-11-15T00:44:45,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631545676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631545680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631545680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,849 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/9f93bdae69434aa1a670bd57c18c40ad as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/9f93bdae69434aa1a670bd57c18c40ad 2024-11-15T00:44:45,875 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/B of c6201d6aebcecefaa2f781f0e095054e into 9f93bdae69434aa1a670bd57c18c40ad(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:45,875 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:45,875 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/B, priority=13, startTime=1731631485201; duration=0sec 2024-11-15T00:44:45,875 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:45,875 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:B 2024-11-15T00:44:45,931 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/1789c40aa2ef4594a6c909099dd3e08c 2024-11-15T00:44:45,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631545970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:45,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631545979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631545992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631546001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631546001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/68cb6054f12547cfae2f034b50d0bd92 is 50, key is test_row_0/C:col10/1731631484557/Put/seqid=0 2024-11-15T00:44:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:44:46,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741863_1039 (size=12151) 2024-11-15T00:44:46,092 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/68cb6054f12547cfae2f034b50d0bd92 2024-11-15T00:44:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/ca930d7bd47746eca9fbaf9275c1ce39 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/ca930d7bd47746eca9fbaf9275c1ce39 2024-11-15T00:44:46,124 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/ca930d7bd47746eca9fbaf9275c1ce39, entries=200, sequenceid=155, filesize=14.2 K 2024-11-15T00:44:46,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/1789c40aa2ef4594a6c909099dd3e08c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1789c40aa2ef4594a6c909099dd3e08c 2024-11-15T00:44:46,144 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1789c40aa2ef4594a6c909099dd3e08c, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T00:44:46,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/68cb6054f12547cfae2f034b50d0bd92 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/68cb6054f12547cfae2f034b50d0bd92 2024-11-15T00:44:46,195 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/68cb6054f12547cfae2f034b50d0bd92, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T00:44:46,197 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c6201d6aebcecefaa2f781f0e095054e in 959ms, sequenceid=155, compaction requested=false 2024-11-15T00:44:46,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:46,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:46,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T00:44:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T00:44:46,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T00:44:46,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2870 sec 2024-11-15T00:44:46,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3180 sec 2024-11-15T00:44:46,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:44:46,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:46,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:46,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:46,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:46,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:46,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:46,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/42539d6fd34e434fa4564e540c339516 is 50, key is test_row_0/A:col10/1731631486503/Put/seqid=0 2024-11-15T00:44:46,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741864_1040 (size=14541) 2024-11-15T00:44:46,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631546590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631546597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631546598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631546599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631546602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631546711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631546750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631546750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631546759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631546759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631546919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631546961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631546970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631546971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:46,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631546980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/42539d6fd34e434fa4564e540c339516 2024-11-15T00:44:47,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/ad3456d182c9477cbabbef36248b59d0 is 50, key is test_row_0/B:col10/1731631486503/Put/seqid=0 2024-11-15T00:44:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:44:47,061 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:44:47,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:44:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T00:44:47,065 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:44:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:44:47,076 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:44:47,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:44:47,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741865_1041 (size=12151) 2024-11-15T00:44:47,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/ad3456d182c9477cbabbef36248b59d0 2024-11-15T00:44:47,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/0a3d3834ed4c4ee3a6f6fb59aa13c16d is 50, key is test_row_0/C:col10/1731631486503/Put/seqid=0 2024-11-15T00:44:47,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741866_1042 (size=12151) 2024-11-15T00:44:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:44:47,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/0a3d3834ed4c4ee3a6f6fb59aa13c16d 2024-11-15T00:44:47,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/42539d6fd34e434fa4564e540c339516 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/42539d6fd34e434fa4564e540c339516 2024-11-15T00:44:47,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/42539d6fd34e434fa4564e540c339516, entries=200, sequenceid=170, filesize=14.2 K 2024-11-15T00:44:47,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/ad3456d182c9477cbabbef36248b59d0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ad3456d182c9477cbabbef36248b59d0 2024-11-15T00:44:47,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ad3456d182c9477cbabbef36248b59d0, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T00:44:47,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/0a3d3834ed4c4ee3a6f6fb59aa13c16d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/0a3d3834ed4c4ee3a6f6fb59aa13c16d 2024-11-15T00:44:47,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/0a3d3834ed4c4ee3a6f6fb59aa13c16d, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T00:44:47,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:44:47,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:47,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c6201d6aebcecefaa2f781f0e095054e in 734ms, sequenceid=170, compaction requested=true 2024-11-15T00:44:47,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:47,242 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:44:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:44:47,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:47,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:44:47,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:47,242 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:44:47,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:44:47,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,253 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41373 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:44:47,253 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/A is initiating minor compaction (all files) 2024-11-15T00:44:47,253 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/A in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,253 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/8e2bef57724e4539835cd815b8fbe66b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/ca930d7bd47746eca9fbaf9275c1ce39, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/42539d6fd34e434fa4564e540c339516] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=40.4 K 2024-11-15T00:44:47,255 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e2bef57724e4539835cd815b8fbe66b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631482780 2024-11-15T00:44:47,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T00:44:47,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:47,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:47,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:47,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:47,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:47,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:47,260 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:44:47,260 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ca930d7bd47746eca9fbaf9275c1ce39, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631484557 2024-11-15T00:44:47,261 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/B is initiating minor compaction (all files) 2024-11-15T00:44:47,261 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/B in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,262 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 42539d6fd34e434fa4564e540c339516, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631485287 2024-11-15T00:44:47,262 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/9f93bdae69434aa1a670bd57c18c40ad, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1789c40aa2ef4594a6c909099dd3e08c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ad3456d182c9477cbabbef36248b59d0] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=35.7 K 2024-11-15T00:44:47,265 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f93bdae69434aa1a670bd57c18c40ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631482780 2024-11-15T00:44:47,268 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1789c40aa2ef4594a6c909099dd3e08c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631484557 2024-11-15T00:44:47,270 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad3456d182c9477cbabbef36248b59d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631485287 2024-11-15T00:44:47,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/3115779b47174aa6ab9f1ac9f84a3a40 is 50, key is test_row_0/A:col10/1731631486596/Put/seqid=0 2024-11-15T00:44:47,327 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:47,327 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/672e64a4319346899cf1ecd55355ed04 is 50, key is test_row_0/A:col10/1731631486503/Put/seqid=0 2024-11-15T00:44:47,332 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#B#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:47,333 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/68c6cca89bcc41638b1147a909cc7c85 is 50, key is test_row_0/B:col10/1731631486503/Put/seqid=0 2024-11-15T00:44:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:44:47,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741868_1044 (size=12493) 2024-11-15T00:44:47,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741867_1043 (size=14541) 2024-11-15T00:44:47,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/3115779b47174aa6ab9f1ac9f84a3a40 2024-11-15T00:44:47,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741869_1045 (size=12493) 2024-11-15T00:44:47,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:44:47,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:47,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,414 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/672e64a4319346899cf1ecd55355ed04 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/672e64a4319346899cf1ecd55355ed04 2024-11-15T00:44:47,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631547392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631547395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631547396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,429 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/68c6cca89bcc41638b1147a909cc7c85 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/68c6cca89bcc41638b1147a909cc7c85 2024-11-15T00:44:47,432 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/A of c6201d6aebcecefaa2f781f0e095054e into 672e64a4319346899cf1ecd55355ed04(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:47,432 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:47,432 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/A, priority=13, startTime=1731631487242; duration=0sec 2024-11-15T00:44:47,432 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:47,432 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:A 2024-11-15T00:44:47,433 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:44:47,435 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:44:47,435 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/C is initiating minor compaction (all files) 2024-11-15T00:44:47,435 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/C in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,436 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/303b7fdd185946bba336e01270efecc6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/68cb6054f12547cfae2f034b50d0bd92, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/0a3d3834ed4c4ee3a6f6fb59aa13c16d] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=35.7 K 2024-11-15T00:44:47,437 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 303b7fdd185946bba336e01270efecc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631482780 2024-11-15T00:44:47,439 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 68cb6054f12547cfae2f034b50d0bd92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631484557 2024-11-15T00:44:47,441 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a3d3834ed4c4ee3a6f6fb59aa13c16d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631485287 2024-11-15T00:44:47,442 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/B of c6201d6aebcecefaa2f781f0e095054e into 68c6cca89bcc41638b1147a909cc7c85(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:47,443 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:47,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,443 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/B, priority=13, startTime=1731631487242; duration=0sec 2024-11-15T00:44:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631547414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,443 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:47,443 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:B 2024-11-15T00:44:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631547420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/1b702c0096804282a7620ec4150e7417 is 50, key is test_row_0/B:col10/1731631486596/Put/seqid=0 2024-11-15T00:44:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,496 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#C#compaction#34 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:47,497 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/354e66de2a754fc6bf06b8cb1f8cbaac is 50, key is test_row_0/C:col10/1731631486503/Put/seqid=0 2024-11-15T00:44:47,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741870_1046 (size=12151) 2024-11-15T00:44:47,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/1b702c0096804282a7620ec4150e7417 2024-11-15T00:44:47,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631547530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631547530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631547530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741871_1047 (size=12493) 2024-11-15T00:44:47,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/78f347a8e27941799e2730c7495f4279 is 50, key is test_row_0/C:col10/1731631486596/Put/seqid=0 2024-11-15T00:44:47,557 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/354e66de2a754fc6bf06b8cb1f8cbaac as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/354e66de2a754fc6bf06b8cb1f8cbaac 2024-11-15T00:44:47,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631547550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,571 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/C of c6201d6aebcecefaa2f781f0e095054e into 354e66de2a754fc6bf06b8cb1f8cbaac(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:47,571 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:47,571 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/C, priority=13, startTime=1731631487242; duration=0sec 2024-11-15T00:44:47,571 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:47,571 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:C 2024-11-15T00:44:47,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:44:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631547560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741872_1048 (size=12151) 2024-11-15T00:44:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:44:47,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:44:47,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:47,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631547740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631547740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631547746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631547772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631547791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:47,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:44:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:47,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:48,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/78f347a8e27941799e2730c7495f4279 2024-11-15T00:44:48,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:44:48,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:48,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:48,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:48,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:48,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:48,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/3115779b47174aa6ab9f1ac9f84a3a40 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3115779b47174aa6ab9f1ac9f84a3a40 2024-11-15T00:44:48,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3115779b47174aa6ab9f1ac9f84a3a40, entries=200, sequenceid=191, filesize=14.2 K 2024-11-15T00:44:48,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/1b702c0096804282a7620ec4150e7417 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1b702c0096804282a7620ec4150e7417 2024-11-15T00:44:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631548061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631548061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1b702c0096804282a7620ec4150e7417, entries=150, sequenceid=191, filesize=11.9 K 2024-11-15T00:44:48,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631548070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/78f347a8e27941799e2730c7495f4279 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/78f347a8e27941799e2730c7495f4279 2024-11-15T00:44:48,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631548110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631548110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/78f347a8e27941799e2730c7495f4279, entries=150, sequenceid=191, filesize=11.9 K 2024-11-15T00:44:48,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for c6201d6aebcecefaa2f781f0e095054e in 887ms, sequenceid=191, compaction requested=false 2024-11-15T00:44:48,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:44:48,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:44:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:48,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:44:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:48,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/528347c31eff4e1a8f7f226b49366a0e is 50, key is test_row_0/A:col10/1731631487405/Put/seqid=0 2024-11-15T00:44:48,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741873_1049 (size=12151) 2024-11-15T00:44:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:48,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:48,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631548646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631548646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631548653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631548657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631548657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,687 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/528347c31eff4e1a8f7f226b49366a0e 2024-11-15T00:44:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/5f30ce404e984a129eaf053177b7bc80 is 50, key is test_row_0/B:col10/1731631487405/Put/seqid=0 2024-11-15T00:44:48,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741874_1050 (size=12151) 2024-11-15T00:44:48,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631548770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631548770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631548770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631548771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631548775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631548990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631548992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631548993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631548993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631548993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:49,152 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/5f30ce404e984a129eaf053177b7bc80 2024-11-15T00:44:49,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:44:49,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/4280c748f1f847c0bec34dff760beac3 is 50, key is test_row_0/C:col10/1731631487405/Put/seqid=0 2024-11-15T00:44:49,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741875_1051 (size=12151) 2024-11-15T00:44:49,310 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/4280c748f1f847c0bec34dff760beac3 2024-11-15T00:44:49,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:49,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631549305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631549313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631549313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:49,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631549314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:49,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631549318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:49,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:49,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/528347c31eff4e1a8f7f226b49366a0e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/528347c31eff4e1a8f7f226b49366a0e 2024-11-15T00:44:49,346 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/528347c31eff4e1a8f7f226b49366a0e, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T00:44:49,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/5f30ce404e984a129eaf053177b7bc80 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f30ce404e984a129eaf053177b7bc80 2024-11-15T00:44:49,368 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f30ce404e984a129eaf053177b7bc80, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T00:44:49,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/4280c748f1f847c0bec34dff760beac3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/4280c748f1f847c0bec34dff760beac3 2024-11-15T00:44:49,403 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/4280c748f1f847c0bec34dff760beac3, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T00:44:49,408 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c6201d6aebcecefaa2f781f0e095054e in 1184ms, sequenceid=206, compaction requested=true 2024-11-15T00:44:49,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:49,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:49,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T00:44:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T00:44:49,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T00:44:49,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3450 sec 2024-11-15T00:44:49,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.3690 sec 2024-11-15T00:44:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:49,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631488630, max timestamp=1731631489820Num uniques 859; 2024-11-15T00:44:49,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:44:49,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631488630, max timestamp=1731631489820Num uniques 859; 2024-11-15T00:44:49,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:44:49,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-15T00:44:49,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:49,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:49,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:49,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:49,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:49,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:49,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/25387331b35c494aae36a79ce12b98ac is 50, key is test_row_0/A:col10/1731631488652/Put/seqid=0 2024-11-15T00:44:49,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631489890, max timestamp=1731631489923Num uniques 859; 2024-11-15T00:44:49,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:44:49,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631489890, max timestamp=1731631489923Num uniques 859; 2024-11-15T00:44:49,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631489890, max timestamp=1731631489923Num uniques 859; 2024-11-15T00:44:49,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:44:49,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:44:49,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741876_1052 (size=12151) 2024-11-15T00:44:49,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/25387331b35c494aae36a79ce12b98ac 2024-11-15T00:44:50,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/bfc3586a23214dfd87d2a328f15749fb is 50, key is test_row_0/B:col10/1731631488652/Put/seqid=0 2024-11-15T00:44:50,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741877_1053 (size=12151) 2024-11-15T00:44:50,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/bfc3586a23214dfd87d2a328f15749fb 2024-11-15T00:44:50,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631550026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631550028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631550032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631550070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/82ecbea4af624a0fb5645ba97a0bac32 is 50, key is test_row_0/C:col10/1731631488652/Put/seqid=0 2024-11-15T00:44:50,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631550089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741878_1054 (size=12151) 2024-11-15T00:44:50,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631550182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631550190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631550182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631550199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631550210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631550401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631550410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631550410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631550419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631550419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/82ecbea4af624a0fb5645ba97a0bac32 2024-11-15T00:44:50,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/25387331b35c494aae36a79ce12b98ac as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/25387331b35c494aae36a79ce12b98ac 2024-11-15T00:44:50,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/25387331b35c494aae36a79ce12b98ac, entries=150, sequenceid=227, filesize=11.9 K 2024-11-15T00:44:50,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/bfc3586a23214dfd87d2a328f15749fb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/bfc3586a23214dfd87d2a328f15749fb 2024-11-15T00:44:50,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/bfc3586a23214dfd87d2a328f15749fb, entries=150, sequenceid=227, filesize=11.9 K 2024-11-15T00:44:50,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/82ecbea4af624a0fb5645ba97a0bac32 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/82ecbea4af624a0fb5645ba97a0bac32 2024-11-15T00:44:50,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/82ecbea4af624a0fb5645ba97a0bac32, entries=150, sequenceid=227, filesize=11.9 K 2024-11-15T00:44:50,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=133.00 KB/136188 for c6201d6aebcecefaa2f781f0e095054e in 815ms, sequenceid=227, compaction requested=true 2024-11-15T00:44:50,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:50,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:44:50,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:50,646 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:50,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:44:50,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:50,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:44:50,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:44:50,646 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:50,650 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:50,650 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/B is initiating minor compaction (all files) 2024-11-15T00:44:50,650 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/B in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:50,650 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/68c6cca89bcc41638b1147a909cc7c85, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1b702c0096804282a7620ec4150e7417, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f30ce404e984a129eaf053177b7bc80, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/bfc3586a23214dfd87d2a328f15749fb] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=47.8 K 2024-11-15T00:44:50,651 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:50,651 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/A is initiating minor compaction (all files) 2024-11-15T00:44:50,651 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 68c6cca89bcc41638b1147a909cc7c85, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631485287 2024-11-15T00:44:50,651 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b702c0096804282a7620ec4150e7417, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731631486596 2024-11-15T00:44:50,652 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/A in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:50,652 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/672e64a4319346899cf1ecd55355ed04, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3115779b47174aa6ab9f1ac9f84a3a40, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/528347c31eff4e1a8f7f226b49366a0e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/25387331b35c494aae36a79ce12b98ac] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=50.1 K 2024-11-15T00:44:50,653 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f30ce404e984a129eaf053177b7bc80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631487329 2024-11-15T00:44:50,653 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 672e64a4319346899cf1ecd55355ed04, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631485287 2024-11-15T00:44:50,654 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3115779b47174aa6ab9f1ac9f84a3a40, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731631486525 2024-11-15T00:44:50,655 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 528347c31eff4e1a8f7f226b49366a0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631487329 2024-11-15T00:44:50,660 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25387331b35c494aae36a79ce12b98ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731631488652 2024-11-15T00:44:50,660 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc3586a23214dfd87d2a328f15749fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731631488652 2024-11-15T00:44:50,692 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:50,693 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/660ba2b4c5394cd29efbee4bf695a358 is 50, key is test_row_0/A:col10/1731631488652/Put/seqid=0 2024-11-15T00:44:50,707 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:50,708 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/d5d7f3492ffc4616bdef2e07f1ea5e9e is 50, key is test_row_0/B:col10/1731631488652/Put/seqid=0 2024-11-15T00:44:50,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731631490027, max timestamp=1731631490718Num uniques 1009; 2024-11-15T00:44:50,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7688, suffix heap size=132904,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-15T00:44:50,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731631490027, max timestamp=1731631490718Num uniques 1009; 2024-11-15T00:44:50,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7688, suffix heap size=132904,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-15T00:44:50,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=64.46 KB heapSize=177.88 KB 2024-11-15T00:44:50,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:50,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:50,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:50,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:50,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:50,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T00:44:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:50,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/aa420f01da744c748bc766ea71ee89ad is 50, key is test_row_0/A:col10/1731631490060/Put/seqid=0 2024-11-15T00:44:50,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741880_1056 (size=12629) 2024-11-15T00:44:50,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741879_1055 (size=12629) 2024-11-15T00:44:50,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741881_1057 (size=12301) 2024-11-15T00:44:50,797 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/d5d7f3492ffc4616bdef2e07f1ea5e9e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/d5d7f3492ffc4616bdef2e07f1ea5e9e 2024-11-15T00:44:50,809 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/B of c6201d6aebcecefaa2f781f0e095054e into d5d7f3492ffc4616bdef2e07f1ea5e9e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:50,809 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:50,809 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/B, priority=12, startTime=1731631490646; duration=0sec 2024-11-15T00:44:50,809 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:50,809 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:B 2024-11-15T00:44:50,809 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:50,812 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:50,812 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/C is initiating minor compaction (all files) 2024-11-15T00:44:50,812 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/C in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:50,813 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/354e66de2a754fc6bf06b8cb1f8cbaac, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/78f347a8e27941799e2730c7495f4279, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/4280c748f1f847c0bec34dff760beac3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/82ecbea4af624a0fb5645ba97a0bac32] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=47.8 K 2024-11-15T00:44:50,814 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 354e66de2a754fc6bf06b8cb1f8cbaac, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631485287 2024-11-15T00:44:50,815 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 78f347a8e27941799e2730c7495f4279, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731631486596 2024-11-15T00:44:50,816 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 4280c748f1f847c0bec34dff760beac3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631487329 2024-11-15T00:44:50,821 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ecbea4af624a0fb5645ba97a0bac32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731631488652 2024-11-15T00:44:50,859 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#C#compaction#45 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:50,860 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/a3cec77922074c62a2cbcdabeb1a8c91 is 50, key is test_row_0/C:col10/1731631488652/Put/seqid=0 2024-11-15T00:44:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631550834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631550840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741882_1058 (size=12629) 2024-11-15T00:44:50,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631550860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631550859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631550861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,925 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/a3cec77922074c62a2cbcdabeb1a8c91 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a3cec77922074c62a2cbcdabeb1a8c91 2024-11-15T00:44:50,963 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/C of c6201d6aebcecefaa2f781f0e095054e into a3cec77922074c62a2cbcdabeb1a8c91(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:50,963 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:50,963 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/C, priority=12, startTime=1731631490646; duration=0sec 2024-11-15T00:44:50,963 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:50,963 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:C 2024-11-15T00:44:50,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631550972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:50,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631550972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631551021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631551021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631551022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/aa420f01da744c748bc766ea71ee89ad 2024-11-15T00:44:51,203 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/660ba2b4c5394cd29efbee4bf695a358 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/660ba2b4c5394cd29efbee4bf695a358 2024-11-15T00:44:51,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631551193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631551193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/4fbabbedaec04113aae3d93da421d840 is 50, key is test_row_0/B:col10/1731631490060/Put/seqid=0 2024-11-15T00:44:51,227 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/A of c6201d6aebcecefaa2f781f0e095054e into 660ba2b4c5394cd29efbee4bf695a358(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:51,227 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:51,227 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/A, priority=12, startTime=1731631490646; duration=0sec 2024-11-15T00:44:51,227 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:51,227 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:A 2024-11-15T00:44:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:44:51,233 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:44:51,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:44:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T00:44:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:44:51,238 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:44:51,239 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:44:51,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:44:51,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631551239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631551240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631551240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741883_1059 (size=12301) 2024-11-15T00:44:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:44:51,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T00:44:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:51,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:51,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631551519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631551529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:44:51,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T00:44:51,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:51,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:51,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:51,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:51,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631551570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631551571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631551571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:51,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/4fbabbedaec04113aae3d93da421d840 2024-11-15T00:44:51,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/15da56351c7841cca7bbd11252036966 is 50, key is test_row_0/C:col10/1731631490060/Put/seqid=0 2024-11-15T00:44:51,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741884_1060 (size=12301) 2024-11-15T00:44:51,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/15da56351c7841cca7bbd11252036966 2024-11-15T00:44:51,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T00:44:51,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:51,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:51,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:51,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:51,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:51,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/aa420f01da744c748bc766ea71ee89ad as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/aa420f01da744c748bc766ea71ee89ad 2024-11-15T00:44:51,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/aa420f01da744c748bc766ea71ee89ad, entries=150, sequenceid=265, filesize=12.0 K 2024-11-15T00:44:51,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/4fbabbedaec04113aae3d93da421d840 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/4fbabbedaec04113aae3d93da421d840 2024-11-15T00:44:51,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/4fbabbedaec04113aae3d93da421d840, entries=150, sequenceid=265, filesize=12.0 K 2024-11-15T00:44:51,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/15da56351c7841cca7bbd11252036966 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/15da56351c7841cca7bbd11252036966 2024-11-15T00:44:51,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/15da56351c7841cca7bbd11252036966, entries=150, sequenceid=265, filesize=12.0 K 2024-11-15T00:44:51,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.46 KB/66006, heapSize ~177.84 KB/182104, currentSize=114.05 KB/116790 for c6201d6aebcecefaa2f781f0e095054e in 1065ms, sequenceid=265, compaction requested=false 2024-11-15T00:44:51,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:44:51,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T00:44:51,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:51,884 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:44:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:51,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:51,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/d95b7581a36f42f4895c8207e9971c7b is 50, key is test_row_0/A:col10/1731631490823/Put/seqid=0 2024-11-15T00:44:51,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741885_1061 (size=12301) 2024-11-15T00:44:51,922 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/d95b7581a36f42f4895c8207e9971c7b 2024-11-15T00:44:51,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/a38bc1b997e04d0f8f3ca03d5651e066 is 50, key is test_row_0/B:col10/1731631490823/Put/seqid=0 2024-11-15T00:44:51,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741886_1062 (size=12301) 2024-11-15T00:44:51,986 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/a38bc1b997e04d0f8f3ca03d5651e066 2024-11-15T00:44:52,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:52,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/5882f9a991204d949f9018e1ecb14d04 is 50, key is test_row_0/C:col10/1731631490823/Put/seqid=0 2024-11-15T00:44:52,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741887_1063 (size=12301) 2024-11-15T00:44:52,099 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/5882f9a991204d949f9018e1ecb14d04 2024-11-15T00:44:52,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631552090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631552090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631552091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631552101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631552105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/d95b7581a36f42f4895c8207e9971c7b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d95b7581a36f42f4895c8207e9971c7b 2024-11-15T00:44:52,129 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d95b7581a36f42f4895c8207e9971c7b, entries=150, sequenceid=288, filesize=12.0 K 2024-11-15T00:44:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/a38bc1b997e04d0f8f3ca03d5651e066 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/a38bc1b997e04d0f8f3ca03d5651e066 2024-11-15T00:44:52,142 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/a38bc1b997e04d0f8f3ca03d5651e066, entries=150, sequenceid=288, filesize=12.0 K 2024-11-15T00:44:52,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/5882f9a991204d949f9018e1ecb14d04 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/5882f9a991204d949f9018e1ecb14d04 2024-11-15T00:44:52,163 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/5882f9a991204d949f9018e1ecb14d04, entries=150, sequenceid=288, filesize=12.0 K 2024-11-15T00:44:52,168 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c6201d6aebcecefaa2f781f0e095054e in 285ms, sequenceid=288, compaction requested=true 2024-11-15T00:44:52,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:52,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:52,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T00:44:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T00:44:52,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T00:44:52,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 944 msec 2024-11-15T00:44:52,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 963 msec 2024-11-15T00:44:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:52,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:44:52,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:52,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:52,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:52,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:52,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:52,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:52,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/14e2efdc054d42449673e3235d5fb700 is 50, key is test_row_0/A:col10/1731631492216/Put/seqid=0 2024-11-15T00:44:52,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741888_1064 (size=14741) 2024-11-15T00:44:52,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/14e2efdc054d42449673e3235d5fb700 2024-11-15T00:44:52,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/3d964c53c06c4bb3943bb9d458b796c2 is 50, key is test_row_0/B:col10/1731631492216/Put/seqid=0 2024-11-15T00:44:52,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631552337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631552347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631552350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:44:52,371 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:44:52,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:44:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T00:44:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:44:52,375 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:44:52,376 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:44:52,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:44:52,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741889_1065 (size=12301) 2024-11-15T00:44:52,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631552459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:44:52,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631552476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631552476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T00:44:52,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:52,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:52,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:52,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:52,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:44:52,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T00:44:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:52,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:52,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631552691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631552690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631552701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:52,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/3d964c53c06c4bb3943bb9d458b796c2 2024-11-15T00:44:52,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T00:44:52,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:52,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:52,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:52,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:52,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:52,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/ef139319d40c49bc948e5f5c9c4f7be5 is 50, key is test_row_0/C:col10/1731631492216/Put/seqid=0 2024-11-15T00:44:52,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741890_1066 (size=12301) 2024-11-15T00:44:52,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/ef139319d40c49bc948e5f5c9c4f7be5 2024-11-15T00:44:52,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/14e2efdc054d42449673e3235d5fb700 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/14e2efdc054d42449673e3235d5fb700 2024-11-15T00:44:52,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/14e2efdc054d42449673e3235d5fb700, entries=200, sequenceid=301, filesize=14.4 K 2024-11-15T00:44:52,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/3d964c53c06c4bb3943bb9d458b796c2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3d964c53c06c4bb3943bb9d458b796c2 2024-11-15T00:44:52,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3d964c53c06c4bb3943bb9d458b796c2, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T00:44:52,934 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-15T00:44:52,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/ef139319d40c49bc948e5f5c9c4f7be5 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ef139319d40c49bc948e5f5c9c4f7be5 2024-11-15T00:44:52,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ef139319d40c49bc948e5f5c9c4f7be5, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T00:44:52,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c6201d6aebcecefaa2f781f0e095054e in 718ms, sequenceid=301, compaction requested=true 2024-11-15T00:44:52,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:44:52,948 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:52,948 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:44:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:44:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:52,950 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:52,950 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:52,950 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/A is initiating minor compaction (all files) 2024-11-15T00:44:52,950 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/B is initiating minor compaction (all files) 2024-11-15T00:44:52,950 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/B in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:52,950 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/d5d7f3492ffc4616bdef2e07f1ea5e9e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/4fbabbedaec04113aae3d93da421d840, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/a38bc1b997e04d0f8f3ca03d5651e066, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3d964c53c06c4bb3943bb9d458b796c2] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=48.4 K 2024-11-15T00:44:52,951 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/A in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:52,951 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/660ba2b4c5394cd29efbee4bf695a358, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/aa420f01da744c748bc766ea71ee89ad, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d95b7581a36f42f4895c8207e9971c7b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/14e2efdc054d42449673e3235d5fb700] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=50.8 K 2024-11-15T00:44:52,951 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting d5d7f3492ffc4616bdef2e07f1ea5e9e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731631488652 2024-11-15T00:44:52,951 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 660ba2b4c5394cd29efbee4bf695a358, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731631488652 2024-11-15T00:44:52,952 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fbabbedaec04113aae3d93da421d840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731631490027 2024-11-15T00:44:52,952 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa420f01da744c748bc766ea71ee89ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731631490027 2024-11-15T00:44:52,952 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting d95b7581a36f42f4895c8207e9971c7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731631490810 2024-11-15T00:44:52,953 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14e2efdc054d42449673e3235d5fb700, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731631492034 2024-11-15T00:44:52,958 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a38bc1b997e04d0f8f3ca03d5651e066, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731631490810 2024-11-15T00:44:52,959 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d964c53c06c4bb3943bb9d458b796c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731631492095 2024-11-15T00:44:52,976 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:52,976 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/be89952bc3754fcd9f6d76176d839144 is 50, key is test_row_0/B:col10/1731631492216/Put/seqid=0 2024-11-15T00:44:52,980 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#A#compaction#54 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:52,981 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/56037362de704bf29650049db08d7bd8 is 50, key is test_row_0/A:col10/1731631492216/Put/seqid=0 2024-11-15T00:44:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:44:53,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T00:44:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:53,007 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T00:44:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:53,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:53,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741892_1068 (size=12915) 2024-11-15T00:44:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:53,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:53,030 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/56037362de704bf29650049db08d7bd8 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/56037362de704bf29650049db08d7bd8 2024-11-15T00:44:53,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/242c212f4360409eb4382235da9477d8 is 50, key is test_row_0/A:col10/1731631492347/Put/seqid=0 2024-11-15T00:44:53,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741891_1067 (size=12915) 2024-11-15T00:44:53,041 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/A of c6201d6aebcecefaa2f781f0e095054e into 56037362de704bf29650049db08d7bd8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:53,041 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:53,041 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/A, priority=12, startTime=1731631492948; duration=0sec 2024-11-15T00:44:53,042 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:53,042 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:A 2024-11-15T00:44:53,042 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:53,044 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:53,049 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/C is initiating minor compaction (all files) 2024-11-15T00:44:53,049 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/C in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:53,049 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a3cec77922074c62a2cbcdabeb1a8c91, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/15da56351c7841cca7bbd11252036966, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/5882f9a991204d949f9018e1ecb14d04, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ef139319d40c49bc948e5f5c9c4f7be5] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=48.4 K 2024-11-15T00:44:53,050 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3cec77922074c62a2cbcdabeb1a8c91, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731631488652 2024-11-15T00:44:53,051 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15da56351c7841cca7bbd11252036966, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731631490027 2024-11-15T00:44:53,051 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5882f9a991204d949f9018e1ecb14d04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731631490810 2024-11-15T00:44:53,056 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef139319d40c49bc948e5f5c9c4f7be5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731631492095 2024-11-15T00:44:53,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741893_1069 (size=12301) 2024-11-15T00:44:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631553059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631553065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631553065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,095 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:53,095 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/16864b3f5ba145c487205fa3db4fd34f is 50, key is test_row_0/C:col10/1731631492216/Put/seqid=0 2024-11-15T00:44:53,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631553109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,119 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in c6201d6aebcecefaa2f781f0e095054e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:44:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631553120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,123 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in c6201d6aebcecefaa2f781f0e095054e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:44:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741894_1070 (size=12915) 2024-11-15T00:44:53,149 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/16864b3f5ba145c487205fa3db4fd34f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/16864b3f5ba145c487205fa3db4fd34f 2024-11-15T00:44:53,162 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/C of c6201d6aebcecefaa2f781f0e095054e into 16864b3f5ba145c487205fa3db4fd34f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:53,162 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:53,162 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/C, priority=12, startTime=1731631492948; duration=0sec 2024-11-15T00:44:53,162 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:53,162 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:C 2024-11-15T00:44:53,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631553169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631553189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631553189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631553379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631553399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631553400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,446 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/be89952bc3754fcd9f6d76176d839144 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/be89952bc3754fcd9f6d76176d839144 2024-11-15T00:44:53,456 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/B of c6201d6aebcecefaa2f781f0e095054e into be89952bc3754fcd9f6d76176d839144(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:53,456 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:53,456 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/B, priority=12, startTime=1731631492948; duration=0sec 2024-11-15T00:44:53,456 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:53,456 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:B 2024-11-15T00:44:53,461 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/242c212f4360409eb4382235da9477d8 2024-11-15T00:44:53,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/3fda2deea8504cac95b01eb141045ff5 is 50, key is test_row_0/B:col10/1731631492347/Put/seqid=0 2024-11-15T00:44:53,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741895_1071 (size=12301) 2024-11-15T00:44:53,503 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/3fda2deea8504cac95b01eb141045ff5 2024-11-15T00:44:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:44:53,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/8f79e8f1baf24731a33047ed3b9edc9e is 50, key is test_row_0/C:col10/1731631492347/Put/seqid=0 2024-11-15T00:44:53,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741896_1072 (size=12301) 2024-11-15T00:44:53,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631553690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631553712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631553712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:53,986 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/8f79e8f1baf24731a33047ed3b9edc9e 2024-11-15T00:44:54,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/242c212f4360409eb4382235da9477d8 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/242c212f4360409eb4382235da9477d8 2024-11-15T00:44:54,012 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/242c212f4360409eb4382235da9477d8, entries=150, sequenceid=320, filesize=12.0 K 2024-11-15T00:44:54,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/3fda2deea8504cac95b01eb141045ff5 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fda2deea8504cac95b01eb141045ff5 2024-11-15T00:44:54,043 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fda2deea8504cac95b01eb141045ff5, entries=150, sequenceid=320, filesize=12.0 K 2024-11-15T00:44:54,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/8f79e8f1baf24731a33047ed3b9edc9e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8f79e8f1baf24731a33047ed3b9edc9e 2024-11-15T00:44:54,063 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8f79e8f1baf24731a33047ed3b9edc9e, entries=150, sequenceid=320, filesize=12.0 K 2024-11-15T00:44:54,068 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c6201d6aebcecefaa2f781f0e095054e in 1061ms, sequenceid=320, compaction requested=false 2024-11-15T00:44:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T00:44:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T00:44:54,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T00:44:54,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7020 sec 2024-11-15T00:44:54,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7240 sec 2024-11-15T00:44:54,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T00:44:54,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:54,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:54,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:54,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:54,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:54,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:54,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/37a1aece0a6c4579acc07c3f816c16d0 is 50, key is test_row_0/A:col10/1731631493046/Put/seqid=0 2024-11-15T00:44:54,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741897_1073 (size=14741) 2024-11-15T00:44:54,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/37a1aece0a6c4579acc07c3f816c16d0 2024-11-15T00:44:54,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/ff87b52483a14c3aa24ebf8b4a60ebde is 50, key is test_row_0/B:col10/1731631493046/Put/seqid=0 2024-11-15T00:44:54,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631554305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631554307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:54,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631554332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:54,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741898_1074 (size=12301) 2024-11-15T00:44:54,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/ff87b52483a14c3aa24ebf8b4a60ebde 2024-11-15T00:44:54,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/ca9fd0b5e5a7483dabd63499b13dfb7e is 50, key is test_row_0/C:col10/1731631493046/Put/seqid=0 2024-11-15T00:44:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741899_1075 (size=12301) 2024-11-15T00:44:54,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631554429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:54,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631554440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631554453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:44:54,530 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:44:54,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:44:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T00:44:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:44:54,534 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:44:54,535 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:44:54,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:44:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:44:54,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631554639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:54,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631554659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:54,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631554669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:54,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T00:44:54,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:54,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:54,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:54,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:54,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:54,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/ca9fd0b5e5a7483dabd63499b13dfb7e 2024-11-15T00:44:54,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/37a1aece0a6c4579acc07c3f816c16d0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/37a1aece0a6c4579acc07c3f816c16d0 2024-11-15T00:44:54,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T00:44:54,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/37a1aece0a6c4579acc07c3f816c16d0, entries=200, sequenceid=337, filesize=14.4 K 2024-11-15T00:44:54,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:54,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:54,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:54,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:54,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:54,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/ff87b52483a14c3aa24ebf8b4a60ebde as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ff87b52483a14c3aa24ebf8b4a60ebde 2024-11-15T00:44:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:44:54,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ff87b52483a14c3aa24ebf8b4a60ebde, entries=150, sequenceid=337, filesize=12.0 K 2024-11-15T00:44:54,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/ca9fd0b5e5a7483dabd63499b13dfb7e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ca9fd0b5e5a7483dabd63499b13dfb7e 2024-11-15T00:44:54,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ca9fd0b5e5a7483dabd63499b13dfb7e, entries=150, sequenceid=337, filesize=12.0 K 2024-11-15T00:44:54,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c6201d6aebcecefaa2f781f0e095054e in 658ms, sequenceid=337, compaction requested=true 2024-11-15T00:44:54,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:54,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:44:54,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:54,870 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:44:54,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:44:54,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:54,870 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:44:54,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:44:54,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:54,872 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:44:54,872 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:44:54,872 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/B is initiating minor compaction (all files) 2024-11-15T00:44:54,872 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/A is initiating minor compaction (all files) 2024-11-15T00:44:54,872 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/B in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:54,872 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/A in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:54,872 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/be89952bc3754fcd9f6d76176d839144, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fda2deea8504cac95b01eb141045ff5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ff87b52483a14c3aa24ebf8b4a60ebde] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=36.6 K 2024-11-15T00:44:54,872 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/56037362de704bf29650049db08d7bd8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/242c212f4360409eb4382235da9477d8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/37a1aece0a6c4579acc07c3f816c16d0] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=39.0 K 2024-11-15T00:44:54,873 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting be89952bc3754fcd9f6d76176d839144, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731631492095 2024-11-15T00:44:54,873 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56037362de704bf29650049db08d7bd8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731631492095 2024-11-15T00:44:54,873 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 242c212f4360409eb4382235da9477d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731631492327 2024-11-15T00:44:54,873 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fda2deea8504cac95b01eb141045ff5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731631492327 2024-11-15T00:44:54,873 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ff87b52483a14c3aa24ebf8b4a60ebde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731631493046 2024-11-15T00:44:54,873 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37a1aece0a6c4579acc07c3f816c16d0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731631493046 2024-11-15T00:44:54,890 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#B#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:54,892 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/07272cb396ed48b8a59bbbf72b98bd7c is 50, key is test_row_0/B:col10/1731631493046/Put/seqid=0 2024-11-15T00:44:54,894 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:54,895 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/3e17b0202f644940b5c206068a3a54f4 is 50, key is test_row_0/A:col10/1731631493046/Put/seqid=0 2024-11-15T00:44:54,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741901_1077 (size=13017) 2024-11-15T00:44:54,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741900_1076 (size=13017) 2024-11-15T00:44:54,951 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/07272cb396ed48b8a59bbbf72b98bd7c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/07272cb396ed48b8a59bbbf72b98bd7c 2024-11-15T00:44:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:54,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T00:44:54,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:54,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:54,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:54,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:54,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:54,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:54,967 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/B of c6201d6aebcecefaa2f781f0e095054e into 07272cb396ed48b8a59bbbf72b98bd7c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:54,967 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:54,968 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/B, priority=13, startTime=1731631494870; duration=0sec 2024-11-15T00:44:54,968 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:54,968 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:B 2024-11-15T00:44:54,968 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:44:54,969 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:44:54,969 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/C is initiating minor compaction (all files) 2024-11-15T00:44:54,969 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/C in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:54,970 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/16864b3f5ba145c487205fa3db4fd34f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8f79e8f1baf24731a33047ed3b9edc9e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ca9fd0b5e5a7483dabd63499b13dfb7e] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=36.6 K 2024-11-15T00:44:54,971 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 16864b3f5ba145c487205fa3db4fd34f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731631492095 2024-11-15T00:44:54,972 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f79e8f1baf24731a33047ed3b9edc9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731631492327 2024-11-15T00:44:54,972 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ca9fd0b5e5a7483dabd63499b13dfb7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731631493046 2024-11-15T00:44:54,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/e1a1e3bb07c24e9d871dad913c2616a4 is 50, key is test_row_0/A:col10/1731631494303/Put/seqid=0 2024-11-15T00:44:54,995 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#C#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:54,996 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/f426cae15dec4f90932c9cf92af457cf is 50, key is test_row_0/C:col10/1731631493046/Put/seqid=0 2024-11-15T00:44:55,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T00:44:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:55,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:55,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:55,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:55,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:55,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631554999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631555000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631555002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741902_1078 (size=12301) 2024-11-15T00:44:55,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/e1a1e3bb07c24e9d871dad913c2616a4 2024-11-15T00:44:55,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741903_1079 (size=13017) 2024-11-15T00:44:55,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/55d15278eea847bd84be0affd292a0c8 is 50, key is test_row_0/B:col10/1731631494303/Put/seqid=0 2024-11-15T00:44:55,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741904_1080 (size=12301) 2024-11-15T00:44:55,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/55d15278eea847bd84be0affd292a0c8 2024-11-15T00:44:55,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/602e1bc7c65b439cac47dc1e1ecc8247 is 50, key is test_row_0/C:col10/1731631494303/Put/seqid=0 2024-11-15T00:44:55,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741905_1081 (size=12301) 2024-11-15T00:44:55,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/602e1bc7c65b439cac47dc1e1ecc8247 2024-11-15T00:44:55,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631555110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631555110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/e1a1e3bb07c24e9d871dad913c2616a4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/e1a1e3bb07c24e9d871dad913c2616a4 2024-11-15T00:44:55,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631555120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/e1a1e3bb07c24e9d871dad913c2616a4, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T00:44:55,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/55d15278eea847bd84be0affd292a0c8 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/55d15278eea847bd84be0affd292a0c8 2024-11-15T00:44:55,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631555139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,144 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in c6201d6aebcecefaa2f781f0e095054e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4285 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:44:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/55d15278eea847bd84be0affd292a0c8, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T00:44:55,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/602e1bc7c65b439cac47dc1e1ecc8247 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/602e1bc7c65b439cac47dc1e1ecc8247 2024-11-15T00:44:55,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631555150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,152 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in c6201d6aebcecefaa2f781f0e095054e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4293 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:44:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T00:44:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:55,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:44:55,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/602e1bc7c65b439cac47dc1e1ecc8247, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T00:44:55,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c6201d6aebcecefaa2f781f0e095054e in 210ms, sequenceid=356, compaction requested=false 2024-11-15T00:44:55,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:55,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T00:44:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:55,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:44:55,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:55,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:55,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:55,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:55,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:55,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:55,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/cf6f576037da469e93cb83b93ae66925 is 50, key is test_row_0/A:col10/1731631494997/Put/seqid=0 2024-11-15T00:44:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:55,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:55,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741906_1082 (size=12301) 2024-11-15T00:44:55,351 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/3e17b0202f644940b5c206068a3a54f4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3e17b0202f644940b5c206068a3a54f4 2024-11-15T00:44:55,361 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/A of c6201d6aebcecefaa2f781f0e095054e into 3e17b0202f644940b5c206068a3a54f4(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:55,361 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:55,361 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/A, priority=13, startTime=1731631494870; duration=0sec 2024-11-15T00:44:55,361 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:55,361 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:A 2024-11-15T00:44:55,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631555357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631555366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631555366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,440 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/f426cae15dec4f90932c9cf92af457cf as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/f426cae15dec4f90932c9cf92af457cf 2024-11-15T00:44:55,450 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/C of c6201d6aebcecefaa2f781f0e095054e into f426cae15dec4f90932c9cf92af457cf(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:55,450 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:55,450 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/C, priority=13, startTime=1731631494871; duration=0sec 2024-11-15T00:44:55,450 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:55,450 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:C 2024-11-15T00:44:55,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631555469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631555479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631555480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:44:55,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631555689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631555689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631555689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:55,756 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/cf6f576037da469e93cb83b93ae66925 2024-11-15T00:44:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/044e151ca19d4d239e480e02c943e32c is 50, key is test_row_0/B:col10/1731631494997/Put/seqid=0 2024-11-15T00:44:55,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741907_1083 (size=12301) 2024-11-15T00:44:56,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631556009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:56,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631556009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:56,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631556009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:56,238 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/044e151ca19d4d239e480e02c943e32c 2024-11-15T00:44:56,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/b3b84165935a486882c9fbfcbae8fd12 is 50, key is test_row_0/C:col10/1731631494997/Put/seqid=0 2024-11-15T00:44:56,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741908_1084 (size=12301) 2024-11-15T00:44:56,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631556521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:56,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:56,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631556522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631556522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:56,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:56,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:44:56,684 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/b3b84165935a486882c9fbfcbae8fd12 2024-11-15T00:44:56,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/cf6f576037da469e93cb83b93ae66925 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/cf6f576037da469e93cb83b93ae66925 2024-11-15T00:44:56,715 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/cf6f576037da469e93cb83b93ae66925, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T00:44:56,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/044e151ca19d4d239e480e02c943e32c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/044e151ca19d4d239e480e02c943e32c 2024-11-15T00:44:56,724 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/044e151ca19d4d239e480e02c943e32c, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T00:44:56,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/b3b84165935a486882c9fbfcbae8fd12 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/b3b84165935a486882c9fbfcbae8fd12 2024-11-15T00:44:56,746 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/b3b84165935a486882c9fbfcbae8fd12, entries=150, sequenceid=370, filesize=12.0 K 2024-11-15T00:44:56,748 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c6201d6aebcecefaa2f781f0e095054e in 1435ms, sequenceid=370, compaction requested=true 2024-11-15T00:44:56,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:56,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:56,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T00:44:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T00:44:56,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T00:44:56,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2190 sec 2024-11-15T00:44:56,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.2380 sec 2024-11-15T00:44:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:57,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:44:57,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631495344, max timestamp=1731631497543Num uniques 859; 2024-11-15T00:44:57,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:44:57,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:57,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:57,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:57,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:57,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:57,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:44:57,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/051f0a9cfd3b4c488560ca870ec008af is 50, key is test_row_0/A:col10/1731631497543/Put/seqid=0 2024-11-15T00:44:57,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741909_1085 (size=12301) 2024-11-15T00:44:57,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631557640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:57,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631557646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:57,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:57,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631557651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:57,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631557759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:57,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631557759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:57,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631557774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:57,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631557969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:57,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631557975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:57,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:57,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:57,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631557990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:58,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/051f0a9cfd3b4c488560ca870ec008af 2024-11-15T00:44:58,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/36ae3f11d27c4cdb8fde366e0034043f is 50, key is test_row_0/B:col10/1731631497543/Put/seqid=0 2024-11-15T00:44:58,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741910_1086 (size=12301) 2024-11-15T00:44:58,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631558299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:58,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631558310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:58,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631558310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:58,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/36ae3f11d27c4cdb8fde366e0034043f 2024-11-15T00:44:58,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/dbf274fdca894d07962677eafaef3457 is 50, key is test_row_0/C:col10/1731631497543/Put/seqid=0 2024-11-15T00:44:58,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741911_1087 (size=12301) 2024-11-15T00:44:58,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/dbf274fdca894d07962677eafaef3457 2024-11-15T00:44:58,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/051f0a9cfd3b4c488560ca870ec008af as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/051f0a9cfd3b4c488560ca870ec008af 2024-11-15T00:44:58,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/051f0a9cfd3b4c488560ca870ec008af, entries=150, sequenceid=394, filesize=12.0 K 2024-11-15T00:44:58,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/36ae3f11d27c4cdb8fde366e0034043f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/36ae3f11d27c4cdb8fde366e0034043f 2024-11-15T00:44:58,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/36ae3f11d27c4cdb8fde366e0034043f, entries=150, sequenceid=394, filesize=12.0 K 2024-11-15T00:44:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/dbf274fdca894d07962677eafaef3457 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/dbf274fdca894d07962677eafaef3457 2024-11-15T00:44:58,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/dbf274fdca894d07962677eafaef3457, entries=150, sequenceid=394, filesize=12.0 K 2024-11-15T00:44:58,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for c6201d6aebcecefaa2f781f0e095054e in 1115ms, sequenceid=394, compaction requested=true 2024-11-15T00:44:58,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:58,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:44:58,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:58,666 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:58,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:44:58,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:58,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6201d6aebcecefaa2f781f0e095054e:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:44:58,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:44:58,666 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:58,668 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:58,668 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:58,668 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/A is initiating minor compaction (all files) 2024-11-15T00:44:58,668 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/B is initiating minor compaction (all files) 2024-11-15T00:44:58,668 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/A in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:58,668 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/B in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:58,669 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3e17b0202f644940b5c206068a3a54f4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/e1a1e3bb07c24e9d871dad913c2616a4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/cf6f576037da469e93cb83b93ae66925, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/051f0a9cfd3b4c488560ca870ec008af] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=48.8 K 2024-11-15T00:44:58,669 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/07272cb396ed48b8a59bbbf72b98bd7c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/55d15278eea847bd84be0affd292a0c8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/044e151ca19d4d239e480e02c943e32c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/36ae3f11d27c4cdb8fde366e0034043f] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=48.8 K 2024-11-15T00:44:58,669 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07272cb396ed48b8a59bbbf72b98bd7c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731631493046 2024-11-15T00:44:58,670 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e17b0202f644940b5c206068a3a54f4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731631493046 2024-11-15T00:44:58,670 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55d15278eea847bd84be0affd292a0c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731631494255 2024-11-15T00:44:58,670 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 044e151ca19d4d239e480e02c943e32c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731631494980 2024-11-15T00:44:58,671 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36ae3f11d27c4cdb8fde366e0034043f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731631495350 2024-11-15T00:44:58,671 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e1a1e3bb07c24e9d871dad913c2616a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731631494255 2024-11-15T00:44:58,671 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting cf6f576037da469e93cb83b93ae66925, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731631494980 2024-11-15T00:44:58,673 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 051f0a9cfd3b4c488560ca870ec008af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731631495350 2024-11-15T00:44:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:44:58,699 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:44:58,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:44:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T00:44:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:44:58,715 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:44:58,715 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#B#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:58,716 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/6c85763a7b0e4113999eeb9fba9f9f28 is 50, key is test_row_0/B:col10/1731631497543/Put/seqid=0 2024-11-15T00:44:58,716 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:44:58,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:44:58,727 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#A#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:58,728 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/e52c325eee7c42eea799b8c662e947ba is 50, key is test_row_0/A:col10/1731631497543/Put/seqid=0 2024-11-15T00:44:58,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741912_1088 (size=13153) 2024-11-15T00:44:58,789 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/6c85763a7b0e4113999eeb9fba9f9f28 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/6c85763a7b0e4113999eeb9fba9f9f28 2024-11-15T00:44:58,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741913_1089 (size=13153) 2024-11-15T00:44:58,810 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/e52c325eee7c42eea799b8c662e947ba as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/e52c325eee7c42eea799b8c662e947ba 2024-11-15T00:44:58,816 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/B of c6201d6aebcecefaa2f781f0e095054e into 6c85763a7b0e4113999eeb9fba9f9f28(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:58,817 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:58,817 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/B, priority=12, startTime=1731631498666; duration=0sec 2024-11-15T00:44:58,817 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:44:58,817 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:B 2024-11-15T00:44:58,817 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:44:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:44:58,823 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:44:58,823 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): c6201d6aebcecefaa2f781f0e095054e/C is initiating minor compaction (all files) 2024-11-15T00:44:58,823 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/A of c6201d6aebcecefaa2f781f0e095054e into e52c325eee7c42eea799b8c662e947ba(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:58,823 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6201d6aebcecefaa2f781f0e095054e/C in TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:58,823 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:58,823 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/A, priority=12, startTime=1731631498666; duration=0sec 2024-11-15T00:44:58,823 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/f426cae15dec4f90932c9cf92af457cf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/602e1bc7c65b439cac47dc1e1ecc8247, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/b3b84165935a486882c9fbfcbae8fd12, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/dbf274fdca894d07962677eafaef3457] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp, totalSize=48.8 K 2024-11-15T00:44:58,823 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:58,824 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:A 2024-11-15T00:44:58,825 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting f426cae15dec4f90932c9cf92af457cf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731631493046 2024-11-15T00:44:58,826 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 602e1bc7c65b439cac47dc1e1ecc8247, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731631494255 2024-11-15T00:44:58,827 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3b84165935a486882c9fbfcbae8fd12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731631494980 2024-11-15T00:44:58,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T00:44:58,827 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbf274fdca894d07962677eafaef3457, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731631495350 2024-11-15T00:44:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:44:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:44:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:44:58,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:44:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:44:58,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/a0cd9773fcee4a8da439845f8293f92c is 50, key is test_row_0/A:col10/1731631498825/Put/seqid=0 2024-11-15T00:44:58,857 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6201d6aebcecefaa2f781f0e095054e#C#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:44:58,860 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/9af9d9833f7145ee97bf31b970ee2dbe is 50, key is test_row_0/C:col10/1731631497543/Put/seqid=0 2024-11-15T00:44:58,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T00:44:58,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:58,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:58,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:58,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:58,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:58,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741915_1091 (size=13153) 2024-11-15T00:44:58,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631558905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:58,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631558908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631558909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:58,918 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/9af9d9833f7145ee97bf31b970ee2dbe as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/9af9d9833f7145ee97bf31b970ee2dbe 2024-11-15T00:44:58,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741914_1090 (size=14741) 2024-11-15T00:44:58,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/a0cd9773fcee4a8da439845f8293f92c 2024-11-15T00:44:58,931 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6201d6aebcecefaa2f781f0e095054e/C of c6201d6aebcecefaa2f781f0e095054e into 9af9d9833f7145ee97bf31b970ee2dbe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:44:58,931 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:44:58,931 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., storeName=c6201d6aebcecefaa2f781f0e095054e/C, priority=12, startTime=1731631498666; duration=0sec 2024-11-15T00:44:58,931 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:44:58,931 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6201d6aebcecefaa2f781f0e095054e:C 2024-11-15T00:44:58,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/d95110c903454365ad987a2c01604903 is 50, key is test_row_0/B:col10/1731631498825/Put/seqid=0 2024-11-15T00:44:59,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741916_1092 (size=12301) 2024-11-15T00:44:59,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631559019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:44:59,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631559019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631559020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T00:44:59,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:59,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54990 deadline: 1731631559189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,199 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in c6201d6aebcecefaa2f781f0e095054e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8340 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:44:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T00:44:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:59,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54974 deadline: 1731631559189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,203 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in c6201d6aebcecefaa2f781f0e095054e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8344 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:44:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631559240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631559240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631559249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:44:59,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T00:44:59,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:59,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/d95110c903454365ad987a2c01604903 2024-11-15T00:44:59,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/06d88775f028436188a5dcab4bf44dbb is 50, key is test_row_0/C:col10/1731631498825/Put/seqid=0 2024-11-15T00:44:59,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T00:44:59,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:59,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741917_1093 (size=12301) 2024-11-15T00:44:59,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54958 deadline: 1731631559549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54928 deadline: 1731631559552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:44:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54970 deadline: 1731631559559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:44:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:44:59,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:44:59,579 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T00:44:59,579 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:44:59,585 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:59,588 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:59,590 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:44:59,592 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T00:44:59,592 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:44:59,593 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:59,593 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:59,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:44:59,596 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T00:44:59,596 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:44:59,596 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:59,596 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:59,597 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T00:44:59,597 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:44:59,597 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:59,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:44:59,597 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:44:59,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:44:59,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T00:44:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:44:59,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T00:44:59,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:44:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:44:59,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:44:59,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/06d88775f028436188a5dcab4bf44dbb 2024-11-15T00:44:59,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/a0cd9773fcee4a8da439845f8293f92c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/a0cd9773fcee4a8da439845f8293f92c 2024-11-15T00:44:59,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/a0cd9773fcee4a8da439845f8293f92c, entries=200, sequenceid=412, filesize=14.4 K 2024-11-15T00:44:59,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/d95110c903454365ad987a2c01604903 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/d95110c903454365ad987a2c01604903 2024-11-15T00:44:59,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/d95110c903454365ad987a2c01604903, entries=150, sequenceid=412, filesize=12.0 K 2024-11-15T00:44:59,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/06d88775f028436188a5dcab4bf44dbb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/06d88775f028436188a5dcab4bf44dbb 2024-11-15T00:44:59,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/06d88775f028436188a5dcab4bf44dbb, entries=150, sequenceid=412, filesize=12.0 K 2024-11-15T00:44:59,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c6201d6aebcecefaa2f781f0e095054e in 1154ms, sequenceid=412, compaction requested=false 2024-11-15T00:44:59,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:45:00,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T00:45:00,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:45:00,012 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T00:45:00,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:45:00,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:00,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:45:00,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:00,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:45:00,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:00,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/07b9e886ab64444d822ea8ce26af9d62 is 50, key is test_row_0/A:col10/1731631498889/Put/seqid=0 2024-11-15T00:45:00,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741918_1094 (size=12301) 2024-11-15T00:45:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:45:00,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. as already flushing 2024-11-15T00:45:00,076 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T00:45:00,076 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:00,076 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:00,076 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:00,076 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:00,083 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T00:45:00,083 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T00:45:00,083 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:00,083 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:00,083 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:00,083 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:00,084 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:00,084 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:00,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:00,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:00,331 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:45:00,430 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/07b9e886ab64444d822ea8ce26af9d62 2024-11-15T00:45:00,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/4ddfb53ac87346ee8fcf297a272e3e35 is 50, key is test_row_0/B:col10/1731631498889/Put/seqid=0 2024-11-15T00:45:00,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741919_1095 (size=12301) 2024-11-15T00:45:00,503 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/4ddfb53ac87346ee8fcf297a272e3e35 2024-11-15T00:45:00,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/331f6478dbbd47c982a0ec1233843669 is 50, key is test_row_0/C:col10/1731631498889/Put/seqid=0 2024-11-15T00:45:00,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741920_1096 (size=12301) 2024-11-15T00:45:00,561 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/331f6478dbbd47c982a0ec1233843669 2024-11-15T00:45:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/07b9e886ab64444d822ea8ce26af9d62 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/07b9e886ab64444d822ea8ce26af9d62 2024-11-15T00:45:00,574 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/07b9e886ab64444d822ea8ce26af9d62, entries=150, sequenceid=430, filesize=12.0 K 2024-11-15T00:45:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/4ddfb53ac87346ee8fcf297a272e3e35 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/4ddfb53ac87346ee8fcf297a272e3e35 2024-11-15T00:45:00,581 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/4ddfb53ac87346ee8fcf297a272e3e35, entries=150, sequenceid=430, filesize=12.0 K 2024-11-15T00:45:00,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/331f6478dbbd47c982a0ec1233843669 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/331f6478dbbd47c982a0ec1233843669 2024-11-15T00:45:00,589 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/331f6478dbbd47c982a0ec1233843669, entries=150, sequenceid=430, filesize=12.0 K 2024-11-15T00:45:00,590 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for c6201d6aebcecefaa2f781f0e095054e in 578ms, sequenceid=430, compaction requested=true 2024-11-15T00:45:00,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c6201d6aebcecefaa2f781f0e095054e: 2024-11-15T00:45:00,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:45:00,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T00:45:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T00:45:00,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T00:45:00,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8810 sec 2024-11-15T00:45:00,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.8970 sec 2024-11-15T00:45:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:45:00,860 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:09,259 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T00:45:09,260 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:09,260 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:09,260 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:09,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:09,261 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T00:45:09,261 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:09,261 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:09,261 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1421 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1410 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 691 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2052 rows 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 698 2024-11-15T00:45:09,262 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2073 rows 2024-11-15T00:45:09,262 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:45:09,262 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:45:09,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:09,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:09,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:09,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:09,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:45:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:45:09,305 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631509305"}]},"ts":"1731631509305"} 2024-11-15T00:45:09,313 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:45:09,313 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:45:09,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:45:09,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6201d6aebcecefaa2f781f0e095054e, UNASSIGN}] 2024-11-15T00:45:09,334 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6201d6aebcecefaa2f781f0e095054e, UNASSIGN 2024-11-15T00:45:09,336 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c6201d6aebcecefaa2f781f0e095054e, regionState=CLOSING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:09,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6201d6aebcecefaa2f781f0e095054e, UNASSIGN because future has completed 2024-11-15T00:45:09,351 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:45:09,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:45:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:45:09,511 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:45:09,511 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:45:09,512 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing c6201d6aebcecefaa2f781f0e095054e, disabling compactions & flushes 2024-11-15T00:45:09,512 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:45:09,512 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:45:09,513 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. after waiting 0 ms 2024-11-15T00:45:09,513 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:45:09,513 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing c6201d6aebcecefaa2f781f0e095054e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T00:45:09,514 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=A 2024-11-15T00:45:09,514 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:09,514 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=B 2024-11-15T00:45:09,514 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:09,514 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c6201d6aebcecefaa2f781f0e095054e, store=C 2024-11-15T00:45:09,515 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:09,526 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/d4e3c4849a3c40c1851f5cffb88fd3c1 is 50, key is test_row_0/A:col10/1731631509252/Put/seqid=0 2024-11-15T00:45:09,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741921_1097 (size=12301) 2024-11-15T00:45:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:45:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:45:09,957 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/d4e3c4849a3c40c1851f5cffb88fd3c1 2024-11-15T00:45:10,000 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/9de6bc5687db43248289c6540913b84e is 50, key is test_row_0/B:col10/1731631509252/Put/seqid=0 2024-11-15T00:45:10,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741922_1098 (size=12301) 2024-11-15T00:45:10,025 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/9de6bc5687db43248289c6540913b84e 2024-11-15T00:45:10,050 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/85ce2131ace0469193e1918fc3545740 is 50, key is test_row_0/C:col10/1731631509252/Put/seqid=0 2024-11-15T00:45:10,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741923_1099 (size=12301) 2024-11-15T00:45:10,114 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/85ce2131ace0469193e1918fc3545740 2024-11-15T00:45:10,146 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/A/d4e3c4849a3c40c1851f5cffb88fd3c1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d4e3c4849a3c40c1851f5cffb88fd3c1 2024-11-15T00:45:10,170 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d4e3c4849a3c40c1851f5cffb88fd3c1, entries=150, sequenceid=438, filesize=12.0 K 2024-11-15T00:45:10,173 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/B/9de6bc5687db43248289c6540913b84e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/9de6bc5687db43248289c6540913b84e 2024-11-15T00:45:10,191 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/9de6bc5687db43248289c6540913b84e, entries=150, sequenceid=438, filesize=12.0 K 2024-11-15T00:45:10,196 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/.tmp/C/85ce2131ace0469193e1918fc3545740 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/85ce2131ace0469193e1918fc3545740 2024-11-15T00:45:10,221 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/85ce2131ace0469193e1918fc3545740, entries=150, sequenceid=438, filesize=12.0 K 2024-11-15T00:45:10,223 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c6201d6aebcecefaa2f781f0e095054e in 710ms, sequenceid=438, compaction requested=true 2024-11-15T00:45:10,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d16343887e6244368ee22e7699e24533, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3293fdbf557b4be2a2c978b08b14cd9b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/1e2462a5743a4de0be815014eab1ab95, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/60a4c9d2ad0b464f9a73859e2dab6921, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/7011b47e83ab46f68d472cb286f6ca7b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/bd7d7f1238b04284bccc5d7b1468543b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/8e2bef57724e4539835cd815b8fbe66b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/c069c0b8a2b0481e85f667cdef2341e1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/ca930d7bd47746eca9fbaf9275c1ce39, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/42539d6fd34e434fa4564e540c339516, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/672e64a4319346899cf1ecd55355ed04, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3115779b47174aa6ab9f1ac9f84a3a40, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/528347c31eff4e1a8f7f226b49366a0e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/660ba2b4c5394cd29efbee4bf695a358, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/25387331b35c494aae36a79ce12b98ac, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/aa420f01da744c748bc766ea71ee89ad, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d95b7581a36f42f4895c8207e9971c7b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/14e2efdc054d42449673e3235d5fb700, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/56037362de704bf29650049db08d7bd8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/242c212f4360409eb4382235da9477d8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/37a1aece0a6c4579acc07c3f816c16d0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3e17b0202f644940b5c206068a3a54f4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/e1a1e3bb07c24e9d871dad913c2616a4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/cf6f576037da469e93cb83b93ae66925, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/051f0a9cfd3b4c488560ca870ec008af] to archive 2024-11-15T00:45:10,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:45:10,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d16343887e6244368ee22e7699e24533 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d16343887e6244368ee22e7699e24533 2024-11-15T00:45:10,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3293fdbf557b4be2a2c978b08b14cd9b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3293fdbf557b4be2a2c978b08b14cd9b 2024-11-15T00:45:10,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/1e2462a5743a4de0be815014eab1ab95 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/1e2462a5743a4de0be815014eab1ab95 2024-11-15T00:45:10,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/60a4c9d2ad0b464f9a73859e2dab6921 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/60a4c9d2ad0b464f9a73859e2dab6921 2024-11-15T00:45:10,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/7011b47e83ab46f68d472cb286f6ca7b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/7011b47e83ab46f68d472cb286f6ca7b 2024-11-15T00:45:10,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/bd7d7f1238b04284bccc5d7b1468543b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/bd7d7f1238b04284bccc5d7b1468543b 2024-11-15T00:45:10,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/8e2bef57724e4539835cd815b8fbe66b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/8e2bef57724e4539835cd815b8fbe66b 2024-11-15T00:45:10,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/c069c0b8a2b0481e85f667cdef2341e1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/c069c0b8a2b0481e85f667cdef2341e1 2024-11-15T00:45:10,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/ca930d7bd47746eca9fbaf9275c1ce39 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/ca930d7bd47746eca9fbaf9275c1ce39 2024-11-15T00:45:10,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/42539d6fd34e434fa4564e540c339516 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/42539d6fd34e434fa4564e540c339516 2024-11-15T00:45:10,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/672e64a4319346899cf1ecd55355ed04 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/672e64a4319346899cf1ecd55355ed04 2024-11-15T00:45:10,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3115779b47174aa6ab9f1ac9f84a3a40 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3115779b47174aa6ab9f1ac9f84a3a40 2024-11-15T00:45:10,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/528347c31eff4e1a8f7f226b49366a0e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/528347c31eff4e1a8f7f226b49366a0e 2024-11-15T00:45:10,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/660ba2b4c5394cd29efbee4bf695a358 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/660ba2b4c5394cd29efbee4bf695a358 2024-11-15T00:45:10,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/25387331b35c494aae36a79ce12b98ac to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/25387331b35c494aae36a79ce12b98ac 2024-11-15T00:45:10,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/aa420f01da744c748bc766ea71ee89ad to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/aa420f01da744c748bc766ea71ee89ad 2024-11-15T00:45:10,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d95b7581a36f42f4895c8207e9971c7b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d95b7581a36f42f4895c8207e9971c7b 2024-11-15T00:45:10,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/14e2efdc054d42449673e3235d5fb700 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/14e2efdc054d42449673e3235d5fb700 2024-11-15T00:45:10,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/56037362de704bf29650049db08d7bd8 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/56037362de704bf29650049db08d7bd8 2024-11-15T00:45:10,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/242c212f4360409eb4382235da9477d8 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/242c212f4360409eb4382235da9477d8 2024-11-15T00:45:10,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/37a1aece0a6c4579acc07c3f816c16d0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/37a1aece0a6c4579acc07c3f816c16d0 2024-11-15T00:45:10,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3e17b0202f644940b5c206068a3a54f4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/3e17b0202f644940b5c206068a3a54f4 2024-11-15T00:45:10,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/e1a1e3bb07c24e9d871dad913c2616a4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/e1a1e3bb07c24e9d871dad913c2616a4 2024-11-15T00:45:10,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/cf6f576037da469e93cb83b93ae66925 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/cf6f576037da469e93cb83b93ae66925 2024-11-15T00:45:10,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/051f0a9cfd3b4c488560ca870ec008af to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/051f0a9cfd3b4c488560ca870ec008af 2024-11-15T00:45:10,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/f98545b0aff14f068e2b329b718d482b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/7af1e4b1f66f4a20b2ad9b012168e65f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f8979eae2f0478a814563f826a9d5a1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/31ea9b072cc846599adba8a7f753e68d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/6723ab6b53374955b5b21962d10efd87, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/2d9a385a822b4a5ebf9bf7e8764821ec, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/9f93bdae69434aa1a670bd57c18c40ad, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fdb1ac96d604ccf91a7ef54881fa753, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1789c40aa2ef4594a6c909099dd3e08c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/68c6cca89bcc41638b1147a909cc7c85, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ad3456d182c9477cbabbef36248b59d0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1b702c0096804282a7620ec4150e7417, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f30ce404e984a129eaf053177b7bc80, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/d5d7f3492ffc4616bdef2e07f1ea5e9e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/bfc3586a23214dfd87d2a328f15749fb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/4fbabbedaec04113aae3d93da421d840, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/a38bc1b997e04d0f8f3ca03d5651e066, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/be89952bc3754fcd9f6d76176d839144, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3d964c53c06c4bb3943bb9d458b796c2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fda2deea8504cac95b01eb141045ff5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/07272cb396ed48b8a59bbbf72b98bd7c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ff87b52483a14c3aa24ebf8b4a60ebde, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/55d15278eea847bd84be0affd292a0c8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/044e151ca19d4d239e480e02c943e32c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/36ae3f11d27c4cdb8fde366e0034043f] to archive 2024-11-15T00:45:10,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:45:10,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/f98545b0aff14f068e2b329b718d482b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/f98545b0aff14f068e2b329b718d482b 2024-11-15T00:45:10,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/7af1e4b1f66f4a20b2ad9b012168e65f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/7af1e4b1f66f4a20b2ad9b012168e65f 2024-11-15T00:45:10,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f8979eae2f0478a814563f826a9d5a1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f8979eae2f0478a814563f826a9d5a1 2024-11-15T00:45:10,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/31ea9b072cc846599adba8a7f753e68d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/31ea9b072cc846599adba8a7f753e68d 2024-11-15T00:45:10,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/6723ab6b53374955b5b21962d10efd87 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/6723ab6b53374955b5b21962d10efd87 2024-11-15T00:45:10,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/2d9a385a822b4a5ebf9bf7e8764821ec to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/2d9a385a822b4a5ebf9bf7e8764821ec 2024-11-15T00:45:10,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/9f93bdae69434aa1a670bd57c18c40ad to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/9f93bdae69434aa1a670bd57c18c40ad 2024-11-15T00:45:10,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fdb1ac96d604ccf91a7ef54881fa753 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fdb1ac96d604ccf91a7ef54881fa753 2024-11-15T00:45:10,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1789c40aa2ef4594a6c909099dd3e08c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1789c40aa2ef4594a6c909099dd3e08c 2024-11-15T00:45:10,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/68c6cca89bcc41638b1147a909cc7c85 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/68c6cca89bcc41638b1147a909cc7c85 2024-11-15T00:45:10,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ad3456d182c9477cbabbef36248b59d0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ad3456d182c9477cbabbef36248b59d0 2024-11-15T00:45:10,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1b702c0096804282a7620ec4150e7417 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/1b702c0096804282a7620ec4150e7417 2024-11-15T00:45:10,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f30ce404e984a129eaf053177b7bc80 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/5f30ce404e984a129eaf053177b7bc80 2024-11-15T00:45:10,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/d5d7f3492ffc4616bdef2e07f1ea5e9e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/d5d7f3492ffc4616bdef2e07f1ea5e9e 2024-11-15T00:45:10,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/bfc3586a23214dfd87d2a328f15749fb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/bfc3586a23214dfd87d2a328f15749fb 2024-11-15T00:45:10,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/4fbabbedaec04113aae3d93da421d840 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/4fbabbedaec04113aae3d93da421d840 2024-11-15T00:45:10,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/a38bc1b997e04d0f8f3ca03d5651e066 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/a38bc1b997e04d0f8f3ca03d5651e066 2024-11-15T00:45:10,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/be89952bc3754fcd9f6d76176d839144 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/be89952bc3754fcd9f6d76176d839144 2024-11-15T00:45:10,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3d964c53c06c4bb3943bb9d458b796c2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3d964c53c06c4bb3943bb9d458b796c2 2024-11-15T00:45:10,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fda2deea8504cac95b01eb141045ff5 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/3fda2deea8504cac95b01eb141045ff5 2024-11-15T00:45:10,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/07272cb396ed48b8a59bbbf72b98bd7c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/07272cb396ed48b8a59bbbf72b98bd7c 2024-11-15T00:45:10,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ff87b52483a14c3aa24ebf8b4a60ebde to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/ff87b52483a14c3aa24ebf8b4a60ebde 2024-11-15T00:45:10,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/55d15278eea847bd84be0affd292a0c8 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/55d15278eea847bd84be0affd292a0c8 2024-11-15T00:45:10,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/044e151ca19d4d239e480e02c943e32c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/044e151ca19d4d239e480e02c943e32c 2024-11-15T00:45:10,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/36ae3f11d27c4cdb8fde366e0034043f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/36ae3f11d27c4cdb8fde366e0034043f 2024-11-15T00:45:10,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/22737661330f4dad895db61a5a5ba53c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8c31cfcb8ce6495c895d570296d689e3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a007d155108c4ab1b0aba4a7238a744a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/6c3bdbd0562c467eb4c097dcf5087a83, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/3ec65758c0aa4e4cba47f1257b67ab38, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/55ab3e4198d64897b15f758037522f99, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/303b7fdd185946bba336e01270efecc6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/fe476462f8084655a461d30c4f1a434f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/68cb6054f12547cfae2f034b50d0bd92, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/354e66de2a754fc6bf06b8cb1f8cbaac, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/0a3d3834ed4c4ee3a6f6fb59aa13c16d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/78f347a8e27941799e2730c7495f4279, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/4280c748f1f847c0bec34dff760beac3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a3cec77922074c62a2cbcdabeb1a8c91, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/82ecbea4af624a0fb5645ba97a0bac32, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/15da56351c7841cca7bbd11252036966, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/5882f9a991204d949f9018e1ecb14d04, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/16864b3f5ba145c487205fa3db4fd34f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ef139319d40c49bc948e5f5c9c4f7be5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8f79e8f1baf24731a33047ed3b9edc9e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/f426cae15dec4f90932c9cf92af457cf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ca9fd0b5e5a7483dabd63499b13dfb7e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/602e1bc7c65b439cac47dc1e1ecc8247, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/b3b84165935a486882c9fbfcbae8fd12, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/dbf274fdca894d07962677eafaef3457] to archive 2024-11-15T00:45:10,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:45:10,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/22737661330f4dad895db61a5a5ba53c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/22737661330f4dad895db61a5a5ba53c 2024-11-15T00:45:10,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8c31cfcb8ce6495c895d570296d689e3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8c31cfcb8ce6495c895d570296d689e3 2024-11-15T00:45:10,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a007d155108c4ab1b0aba4a7238a744a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a007d155108c4ab1b0aba4a7238a744a 2024-11-15T00:45:10,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/6c3bdbd0562c467eb4c097dcf5087a83 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/6c3bdbd0562c467eb4c097dcf5087a83 2024-11-15T00:45:10,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/3ec65758c0aa4e4cba47f1257b67ab38 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/3ec65758c0aa4e4cba47f1257b67ab38 2024-11-15T00:45:10,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/55ab3e4198d64897b15f758037522f99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/55ab3e4198d64897b15f758037522f99 2024-11-15T00:45:10,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/303b7fdd185946bba336e01270efecc6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/303b7fdd185946bba336e01270efecc6 2024-11-15T00:45:10,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/fe476462f8084655a461d30c4f1a434f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/fe476462f8084655a461d30c4f1a434f 2024-11-15T00:45:10,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/68cb6054f12547cfae2f034b50d0bd92 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/68cb6054f12547cfae2f034b50d0bd92 2024-11-15T00:45:10,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/354e66de2a754fc6bf06b8cb1f8cbaac to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/354e66de2a754fc6bf06b8cb1f8cbaac 2024-11-15T00:45:10,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/0a3d3834ed4c4ee3a6f6fb59aa13c16d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/0a3d3834ed4c4ee3a6f6fb59aa13c16d 2024-11-15T00:45:10,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/78f347a8e27941799e2730c7495f4279 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/78f347a8e27941799e2730c7495f4279 2024-11-15T00:45:10,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/4280c748f1f847c0bec34dff760beac3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/4280c748f1f847c0bec34dff760beac3 2024-11-15T00:45:10,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a3cec77922074c62a2cbcdabeb1a8c91 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/a3cec77922074c62a2cbcdabeb1a8c91 2024-11-15T00:45:10,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/82ecbea4af624a0fb5645ba97a0bac32 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/82ecbea4af624a0fb5645ba97a0bac32 2024-11-15T00:45:10,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/15da56351c7841cca7bbd11252036966 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/15da56351c7841cca7bbd11252036966 2024-11-15T00:45:10,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/5882f9a991204d949f9018e1ecb14d04 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/5882f9a991204d949f9018e1ecb14d04 2024-11-15T00:45:10,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/16864b3f5ba145c487205fa3db4fd34f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/16864b3f5ba145c487205fa3db4fd34f 2024-11-15T00:45:10,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ef139319d40c49bc948e5f5c9c4f7be5 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ef139319d40c49bc948e5f5c9c4f7be5 2024-11-15T00:45:10,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8f79e8f1baf24731a33047ed3b9edc9e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/8f79e8f1baf24731a33047ed3b9edc9e 2024-11-15T00:45:10,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/f426cae15dec4f90932c9cf92af457cf to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/f426cae15dec4f90932c9cf92af457cf 2024-11-15T00:45:10,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ca9fd0b5e5a7483dabd63499b13dfb7e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/ca9fd0b5e5a7483dabd63499b13dfb7e 2024-11-15T00:45:10,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/602e1bc7c65b439cac47dc1e1ecc8247 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/602e1bc7c65b439cac47dc1e1ecc8247 2024-11-15T00:45:10,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/b3b84165935a486882c9fbfcbae8fd12 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/b3b84165935a486882c9fbfcbae8fd12 2024-11-15T00:45:10,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/dbf274fdca894d07962677eafaef3457 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/dbf274fdca894d07962677eafaef3457 2024-11-15T00:45:10,414 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/recovered.edits/441.seqid, newMaxSeqId=441, maxSeqId=1 2024-11-15T00:45:10,415 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. 2024-11-15T00:45:10,416 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for c6201d6aebcecefaa2f781f0e095054e: Waiting for close lock at 1731631509512Running coprocessor pre-close hooks at 1731631509512Disabling compacts and flushes for region at 1731631509512Disabling writes for close at 1731631509513 (+1 ms)Obtaining lock to block concurrent updates at 1731631509513Preparing flush snapshotting stores in c6201d6aebcecefaa2f781f0e095054e at 1731631509513Finished memstore snapshotting TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731631509515 (+2 ms)Flushing stores of TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e. at 1731631509515Flushing c6201d6aebcecefaa2f781f0e095054e/A: creating writer at 1731631509515Flushing c6201d6aebcecefaa2f781f0e095054e/A: appending metadata at 1731631509526 (+11 ms)Flushing c6201d6aebcecefaa2f781f0e095054e/A: closing flushed file at 1731631509526Flushing c6201d6aebcecefaa2f781f0e095054e/B: creating writer at 1731631509986 (+460 ms)Flushing c6201d6aebcecefaa2f781f0e095054e/B: appending metadata at 1731631509998 (+12 ms)Flushing c6201d6aebcecefaa2f781f0e095054e/B: closing flushed file at 1731631509998Flushing c6201d6aebcecefaa2f781f0e095054e/C: creating writer at 1731631510033 (+35 ms)Flushing c6201d6aebcecefaa2f781f0e095054e/C: appending metadata at 1731631510050 (+17 ms)Flushing c6201d6aebcecefaa2f781f0e095054e/C: closing flushed file at 1731631510050Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59cee76e: reopening flushed file at 1731631510140 (+90 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b5ddea3: reopening flushed file at 1731631510170 (+30 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cd6eb1c: reopening flushed file at 1731631510191 (+21 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c6201d6aebcecefaa2f781f0e095054e in 710ms, sequenceid=438, compaction requested=true at 1731631510223 (+32 ms)Writing region close event to WAL at 1731631510410 (+187 ms)Running coprocessor post-close hooks at 1731631510415 (+5 ms)Closed at 1731631510415 2024-11-15T00:45:10,418 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:45:10,419 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c6201d6aebcecefaa2f781f0e095054e, regionState=CLOSED 2024-11-15T00:45:10,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:45:10,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T00:45:10,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure c6201d6aebcecefaa2f781f0e095054e, server=fb2f6cb7846a,34537,1731631472208 in 1.0710 sec 2024-11-15T00:45:10,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-15T00:45:10,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6201d6aebcecefaa2f781f0e095054e, UNASSIGN in 1.0930 sec 2024-11-15T00:45:10,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T00:45:10,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1060 sec 2024-11-15T00:45:10,431 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631510431"}]},"ts":"1731631510431"} 2024-11-15T00:45:10,433 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:45:10,433 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:45:10,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-15T00:45:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:45:10,450 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:10,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:45:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:10,459 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T00:45:10,461 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:10,466 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:45:10,470 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/recovered.edits] 2024-11-15T00:45:10,472 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/07b9e886ab64444d822ea8ce26af9d62 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/07b9e886ab64444d822ea8ce26af9d62 2024-11-15T00:45:10,474 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/a0cd9773fcee4a8da439845f8293f92c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/a0cd9773fcee4a8da439845f8293f92c 2024-11-15T00:45:10,475 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d4e3c4849a3c40c1851f5cffb88fd3c1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/d4e3c4849a3c40c1851f5cffb88fd3c1 2024-11-15T00:45:10,505 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/e52c325eee7c42eea799b8c662e947ba to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/A/e52c325eee7c42eea799b8c662e947ba 2024-11-15T00:45:10,509 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/4ddfb53ac87346ee8fcf297a272e3e35 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/4ddfb53ac87346ee8fcf297a272e3e35 2024-11-15T00:45:10,510 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/6c85763a7b0e4113999eeb9fba9f9f28 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/6c85763a7b0e4113999eeb9fba9f9f28 2024-11-15T00:45:10,511 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/9de6bc5687db43248289c6540913b84e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/9de6bc5687db43248289c6540913b84e 2024-11-15T00:45:10,513 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/d95110c903454365ad987a2c01604903 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/B/d95110c903454365ad987a2c01604903 2024-11-15T00:45:10,516 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/06d88775f028436188a5dcab4bf44dbb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/06d88775f028436188a5dcab4bf44dbb 2024-11-15T00:45:10,517 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/331f6478dbbd47c982a0ec1233843669 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/331f6478dbbd47c982a0ec1233843669 2024-11-15T00:45:10,518 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/85ce2131ace0469193e1918fc3545740 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/85ce2131ace0469193e1918fc3545740 2024-11-15T00:45:10,519 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/9af9d9833f7145ee97bf31b970ee2dbe to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/C/9af9d9833f7145ee97bf31b970ee2dbe 2024-11-15T00:45:10,522 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/recovered.edits/441.seqid to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e/recovered.edits/441.seqid 2024-11-15T00:45:10,523 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/c6201d6aebcecefaa2f781f0e095054e 2024-11-15T00:45:10,523 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:45:10,529 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:10,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T00:45:10,542 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:45:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T00:45:10,593 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:45:10,595 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:10,595 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:45:10,596 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631510595"}]},"ts":"9223372036854775807"} 2024-11-15T00:45:10,601 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:45:10,601 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c6201d6aebcecefaa2f781f0e095054e, NAME => 'TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:45:10,601 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:45:10,601 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631510601"}]},"ts":"9223372036854775807"} 2024-11-15T00:45:10,605 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:45:10,606 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:10,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 152 msec 2024-11-15T00:45:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T00:45:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:45:10,781 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:10,795 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1114251894_22 at /127.0.0.1:54166 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fb2f6cb7846a:34537-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=894 (was 534) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1983 (was 2940) 2024-11-15T00:45:10,804 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=894, ProcessCount=11, AvailableMemoryMB=1982 2024-11-15T00:45:10,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:45:10,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:45:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:10,815 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:45:10,816 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:10,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-15T00:45:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T00:45:10,817 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:45:10,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741924_1100 (size=1061) 2024-11-15T00:45:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T00:45:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T00:45:11,237 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403 2024-11-15T00:45:11,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741925_1101 (size=53) 2024-11-15T00:45:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T00:45:11,677 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:45:11,677 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f762865d4d0d37fcd928550c43e0b373, disabling compactions & flushes 2024-11-15T00:45:11,677 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:11,677 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:11,677 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. after waiting 0 ms 2024-11-15T00:45:11,677 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:11,677 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:11,677 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f762865d4d0d37fcd928550c43e0b373: Waiting for close lock at 1731631511677Disabling compacts and flushes for region at 1731631511677Disabling writes for close at 1731631511677Writing region close event to WAL at 1731631511677Closed at 1731631511677 2024-11-15T00:45:11,682 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:45:11,683 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631511682"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631511682"}]},"ts":"1731631511682"} 2024-11-15T00:45:11,685 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:45:11,686 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:45:11,686 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631511686"}]},"ts":"1731631511686"} 2024-11-15T00:45:11,688 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:45:11,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, ASSIGN}] 2024-11-15T00:45:11,691 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, ASSIGN 2024-11-15T00:45:11,693 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,34537,1731631472208; forceNewPlan=false, retain=false 2024-11-15T00:45:11,742 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T00:45:11,844 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f762865d4d0d37fcd928550c43e0b373, regionState=OPENING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:11,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, ASSIGN because future has completed 2024-11-15T00:45:11,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:45:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T00:45:12,003 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:12,003 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:45:12,004 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,004 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:45:12,004 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,004 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,006 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,007 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:45:12,008 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f762865d4d0d37fcd928550c43e0b373 columnFamilyName A 2024-11-15T00:45:12,009 DEBUG [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:12,009 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(327): Store=f762865d4d0d37fcd928550c43e0b373/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:45:12,010 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,011 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:45:12,012 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f762865d4d0d37fcd928550c43e0b373 columnFamilyName B 2024-11-15T00:45:12,012 DEBUG [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:12,012 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(327): Store=f762865d4d0d37fcd928550c43e0b373/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:45:12,013 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,014 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:45:12,014 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f762865d4d0d37fcd928550c43e0b373 columnFamilyName C 2024-11-15T00:45:12,014 DEBUG [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:12,015 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(327): Store=f762865d4d0d37fcd928550c43e0b373/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:45:12,015 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:12,015 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,016 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,016 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,017 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,017 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,018 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:45:12,019 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,022 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:45:12,022 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened f762865d4d0d37fcd928550c43e0b373; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70281398, jitterRate=0.04727444052696228}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:45:12,022 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:12,024 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for f762865d4d0d37fcd928550c43e0b373: Running coprocessor pre-open hook at 1731631512004Writing region info on filesystem at 1731631512004Initializing all the Stores at 1731631512005 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631512005Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631512006 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631512006Cleaning up temporary data from old regions at 1731631512017 (+11 ms)Running coprocessor post-open hooks at 1731631512022 (+5 ms)Region opened successfully at 1731631512023 (+1 ms) 2024-11-15T00:45:12,025 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., pid=30, masterSystemTime=1731631511999 2024-11-15T00:45:12,027 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:12,027 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:12,028 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f762865d4d0d37fcd928550c43e0b373, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:12,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:45:12,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-15T00:45:12,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 in 192 msec 2024-11-15T00:45:12,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-15T00:45:12,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, ASSIGN in 353 msec 2024-11-15T00:45:12,046 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:45:12,047 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631512046"}]},"ts":"1731631512046"} 2024-11-15T00:45:12,049 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:45:12,050 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:45:12,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2440 sec 2024-11-15T00:45:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T00:45:12,959 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:12,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244f1d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:12,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:12,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:12,968 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33950ad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:12,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:12,970 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f1c755, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:12,972 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:12,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:12,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:12,982 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:45:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,33521,1731631471305 2024-11-15T00:45:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ad5e6af 2024-11-15T00:45:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:45:12,985 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:45:12,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:45:12,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:45:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:13,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741926_1102 (size=1097) 2024-11-15T00:45:13,432 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T00:45:13,432 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T00:45:13,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:45:13,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, REOPEN/MOVE}] 2024-11-15T00:45:13,450 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, REOPEN/MOVE 2024-11-15T00:45:13,451 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f762865d4d0d37fcd928550c43e0b373, regionState=CLOSING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:13,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, REOPEN/MOVE because future has completed 2024-11-15T00:45:13,456 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:45:13,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:45:13,609 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,609 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:45:13,609 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing f762865d4d0d37fcd928550c43e0b373, disabling compactions & flushes 2024-11-15T00:45:13,609 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:13,610 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:13,610 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. after waiting 0 ms 2024-11-15T00:45:13,610 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:13,614 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:45:13,614 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:13,614 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for f762865d4d0d37fcd928550c43e0b373: Waiting for close lock at 1731631513609Running coprocessor pre-close hooks at 1731631513609Disabling compacts and flushes for region at 1731631513609Disabling writes for close at 1731631513610 (+1 ms)Writing region close event to WAL at 1731631513610Running coprocessor post-close hooks at 1731631513614 (+4 ms)Closed at 1731631513614 2024-11-15T00:45:13,615 WARN [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: f762865d4d0d37fcd928550c43e0b373 to self. 2024-11-15T00:45:13,617 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,618 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f762865d4d0d37fcd928550c43e0b373, regionState=CLOSED 2024-11-15T00:45:13,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:45:13,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T00:45:13,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 in 165 msec 2024-11-15T00:45:13,625 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, REOPEN/MOVE; state=CLOSED, location=fb2f6cb7846a,34537,1731631472208; forceNewPlan=false, retain=true 2024-11-15T00:45:13,776 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f762865d4d0d37fcd928550c43e0b373, regionState=OPENING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:13,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, REOPEN/MOVE because future has completed 2024-11-15T00:45:13,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:45:13,935 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:13,935 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:45:13,935 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,935 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:45:13,935 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,935 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,939 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,940 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:45:13,945 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f762865d4d0d37fcd928550c43e0b373 columnFamilyName A 2024-11-15T00:45:13,949 DEBUG [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:13,950 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(327): Store=f762865d4d0d37fcd928550c43e0b373/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:45:13,950 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,951 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:45:13,951 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f762865d4d0d37fcd928550c43e0b373 columnFamilyName B 2024-11-15T00:45:13,952 DEBUG [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:13,952 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(327): Store=f762865d4d0d37fcd928550c43e0b373/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:45:13,952 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,953 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:45:13,953 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f762865d4d0d37fcd928550c43e0b373 columnFamilyName C 2024-11-15T00:45:13,953 DEBUG [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:13,955 INFO [StoreOpener-f762865d4d0d37fcd928550c43e0b373-1 {}] regionserver.HStore(327): Store=f762865d4d0d37fcd928550c43e0b373/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:45:13,955 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:13,955 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,956 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,961 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,962 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,962 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,963 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:45:13,964 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,966 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened f762865d4d0d37fcd928550c43e0b373; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66024191, jitterRate=-0.01616288721561432}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:45:13,966 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:13,968 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for f762865d4d0d37fcd928550c43e0b373: Running coprocessor pre-open hook at 1731631513935Writing region info on filesystem at 1731631513936 (+1 ms)Initializing all the Stores at 1731631513936Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631513937 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631513937Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631513937Cleaning up temporary data from old regions at 1731631513962 (+25 ms)Running coprocessor post-open hooks at 1731631513966 (+4 ms)Region opened successfully at 1731631513968 (+2 ms) 2024-11-15T00:45:13,969 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., pid=35, masterSystemTime=1731631513931 2024-11-15T00:45:13,973 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:13,973 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:13,975 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f762865d4d0d37fcd928550c43e0b373, regionState=OPEN, openSeqNum=5, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:13,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:45:13,983 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fb2f6cb7846a,34537,1731631472208, table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:45:13,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-15T00:45:13,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 in 205 msec 2024-11-15T00:45:13,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-15T00:45:13,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, REOPEN/MOVE in 538 msec 2024-11-15T00:45:13,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-15T00:45:13,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 554 msec 2024-11-15T00:45:14,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0020 sec 2024-11-15T00:45:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T00:45:14,003 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:14,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b4ffc72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:14,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:14,005 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f63627c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:14,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f90a6ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:14,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:14,013 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cd356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:14,015 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:14,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663f2777, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:14,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:14,017 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:14,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:14,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:14,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42abc2e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:14,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:14,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:14,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3541a4f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:14,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:14,023 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9c3ecb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:14,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:14,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421aa0e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:14,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:14,027 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d32f779, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:14,029 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67caaa6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:14,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:14,033 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11998705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:14,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:14,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:14,036 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192529e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:14,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:14,039 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9eab63d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:14,042 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621343d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:14,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:14,046 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79aee5b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:14,048 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad4bff6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:14,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:14,050 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2133ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:14,052 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62990a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:14,055 DEBUG [Thread-551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:14,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:14,056 DEBUG [Thread-555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:14,056 DEBUG [Thread-553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:14,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:14,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:14,057 DEBUG [Thread-542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-15T00:45:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:14,058 DEBUG [Thread-548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T00:45:14,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:14,059 DEBUG [Thread-540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:14,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:14,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:14,061 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:14,061 DEBUG [Thread-544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:14,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:14,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:14,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:14,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:14,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:45:14,064 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:14,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:14,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:45:14,066 DEBUG [Thread-546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:14,067 DEBUG [Thread-557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:14,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:14,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:45:14,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:14,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:14,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:45:14,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:45:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:45:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:45:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:45:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:45:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:14,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:45:14,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T00:45:14,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111573ef4772c86e4a77b2889125224b8260_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631514129/Put/seqid=0 2024-11-15T00:45:14,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741927_1103 (size=12154) 2024-11-15T00:45:14,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T00:45:14,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:14,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:14,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:14,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631574251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631514237, max timestamp=1731631514262Num uniques 859; 2024-11-15T00:45:14,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:14,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631514237, max timestamp=1731631514262Num uniques 859; 2024-11-15T00:45:14,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:14,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631514246, max timestamp=1731631514262Num uniques 859; 2024-11-15T00:45:14,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:45:14,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631574319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631574365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631574365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631574365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T00:45:14,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:14,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:14,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T00:45:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631574379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631574470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631574482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631574482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631574482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T00:45:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:14,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:14,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631574589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,590 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,611 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111573ef4772c86e4a77b2889125224b8260_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111573ef4772c86e4a77b2889125224b8260_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:14,616 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/90544f73eb2e4b0aa3300c8436a958ec, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:14,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/90544f73eb2e4b0aa3300c8436a958ec is 175, key is test_row_0/A:col10/1731631514129/Put/seqid=0 2024-11-15T00:45:14,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741928_1104 (size=30955) 2024-11-15T00:45:14,653 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/90544f73eb2e4b0aa3300c8436a958ec 2024-11-15T00:45:14,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631574680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T00:45:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T00:45:14,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:14,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631574690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631574690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631574689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:14,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2d5eaa105f064bfeabba99c59515d573 is 50, key is test_row_0/B:col10/1731631514129/Put/seqid=0 2024-11-15T00:45:14,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741929_1105 (size=12001) 2024-11-15T00:45:14,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2d5eaa105f064bfeabba99c59515d573 2024-11-15T00:45:14,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/6b41657ed28f415fa13a8d86538f2aac is 50, key is test_row_0/C:col10/1731631514129/Put/seqid=0 2024-11-15T00:45:14,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741930_1106 (size=12001) 2024-11-15T00:45:14,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/6b41657ed28f415fa13a8d86538f2aac 2024-11-15T00:45:14,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/90544f73eb2e4b0aa3300c8436a958ec as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/90544f73eb2e4b0aa3300c8436a958ec 2024-11-15T00:45:14,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/90544f73eb2e4b0aa3300c8436a958ec, entries=150, sequenceid=16, filesize=30.2 K 2024-11-15T00:45:14,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2d5eaa105f064bfeabba99c59515d573 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2d5eaa105f064bfeabba99c59515d573 2024-11-15T00:45:14,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2d5eaa105f064bfeabba99c59515d573, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T00:45:14,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/6b41657ed28f415fa13a8d86538f2aac as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/6b41657ed28f415fa13a8d86538f2aac 2024-11-15T00:45:14,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/6b41657ed28f415fa13a8d86538f2aac, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T00:45:14,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=136.02 KB/139285 for f762865d4d0d37fcd928550c43e0b373 in 655ms, sequenceid=16, compaction requested=false 2024-11-15T00:45:14,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:14,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T00:45:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:14,842 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=136.02 KB heapSize=384.46 KB 2024-11-15T00:45:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:14,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155e705829d469487883542458d11c0055_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631514325/Put/seqid=0 2024-11-15T00:45:14,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741931_1107 (size=12154) 2024-11-15T00:45:14,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:14,874 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155e705829d469487883542458d11c0055_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155e705829d469487883542458d11c0055_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:14,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/7fe24db973df4c8a82cc4233ca6e52bb, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/7fe24db973df4c8a82cc4233ca6e52bb is 175, key is test_row_0/A:col10/1731631514325/Put/seqid=0 2024-11-15T00:45:14,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741932_1108 (size=30955) 2024-11-15T00:45:14,928 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/7fe24db973df4c8a82cc4233ca6e52bb 2024-11-15T00:45:14,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:14,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/10519e3d8bce4b279fa6ca2f1a08a64a is 50, key is test_row_0/B:col10/1731631514325/Put/seqid=0 2024-11-15T00:45:15,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631574992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741933_1109 (size=12001) 2024-11-15T00:45:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,002 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/10519e3d8bce4b279fa6ca2f1a08a64a 2024-11-15T00:45:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631574997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631574999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631574999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631574999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/de29f95e193842e295c6e7bae36d298b is 50, key is test_row_0/C:col10/1731631514325/Put/seqid=0 2024-11-15T00:45:15,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741934_1110 (size=12001) 2024-11-15T00:45:15,111 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/de29f95e193842e295c6e7bae36d298b 2024-11-15T00:45:15,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631575111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631575111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/7fe24db973df4c8a82cc4233ca6e52bb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/7fe24db973df4c8a82cc4233ca6e52bb 2024-11-15T00:45:15,140 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/7fe24db973df4c8a82cc4233ca6e52bb, entries=150, sequenceid=52, filesize=30.2 K 2024-11-15T00:45:15,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/10519e3d8bce4b279fa6ca2f1a08a64a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/10519e3d8bce4b279fa6ca2f1a08a64a 2024-11-15T00:45:15,149 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/10519e3d8bce4b279fa6ca2f1a08a64a, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T00:45:15,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/de29f95e193842e295c6e7bae36d298b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/de29f95e193842e295c6e7bae36d298b 2024-11-15T00:45:15,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/de29f95e193842e295c6e7bae36d298b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T00:45:15,170 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139285, heapSize ~384.41 KB/393640, currentSize=46.96 KB/48090 for f762865d4d0d37fcd928550c43e0b373 in 327ms, sequenceid=52, compaction requested=false 2024-11-15T00:45:15,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:15,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-15T00:45:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-15T00:45:15,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-15T00:45:15,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1090 sec 2024-11-15T00:45:15,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.1260 sec 2024-11-15T00:45:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T00:45:15,200 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:15,201 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-15T00:45:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T00:45:15,204 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:15,205 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:15,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T00:45:15,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T00:45:15,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:15,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:15,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:15,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:15,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:15,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:15,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:15,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:15,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115714fd1768ec844c8b36000c9508a90c0_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631515335/Put/seqid=0 2024-11-15T00:45:15,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631515380, max timestamp=1731631515445Num uniques 859; 2024-11-15T00:45:15,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:15,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741935_1111 (size=14594) 2024-11-15T00:45:15,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631515380, max timestamp=1731631515445Num uniques 859; 2024-11-15T00:45:15,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:15,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631575445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631515380, max timestamp=1731631515445Num uniques 859; 2024-11-15T00:45:15,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:15,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:15,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T00:45:15,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631575555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631575555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631575557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631575559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631575579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631575670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631575679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631575679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631575680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631575800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T00:45:15,859 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:15,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115714fd1768ec844c8b36000c9508a90c0_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115714fd1768ec844c8b36000c9508a90c0_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:15,876 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/a83af7d89a154178bfdf255b841dc8dd, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:15,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/a83af7d89a154178bfdf255b841dc8dd is 175, key is test_row_0/A:col10/1731631515335/Put/seqid=0 2024-11-15T00:45:15,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631575880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631575889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631575900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631575904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:15,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:15,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741936_1112 (size=39549) 2024-11-15T00:45:15,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:15,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,009 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:45:16,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631576119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:16,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:16,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:16,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631576200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:16,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631576204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:16,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631576219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:16,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631576220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:16,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:16,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:16,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T00:45:16,370 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/a83af7d89a154178bfdf255b841dc8dd 2024-11-15T00:45:16,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2776a135c6b0473aaf31711e905f819b is 50, key is test_row_0/B:col10/1731631515335/Put/seqid=0 2024-11-15T00:45:16,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:16,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:16,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741937_1113 (size=12001) 2024-11-15T00:45:16,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:16,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:16,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631576630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:16,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631576719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:16,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631576721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631576739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631576741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:16,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:16,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:16,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2776a135c6b0473aaf31711e905f819b 2024-11-15T00:45:16,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/14d1cd70f88c4ae79aaa1933b5936bf5 is 50, key is test_row_0/C:col10/1731631515335/Put/seqid=0 2024-11-15T00:45:16,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741938_1114 (size=12001) 2024-11-15T00:45:16,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/14d1cd70f88c4ae79aaa1933b5936bf5 2024-11-15T00:45:16,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/a83af7d89a154178bfdf255b841dc8dd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/a83af7d89a154178bfdf255b841dc8dd 2024-11-15T00:45:16,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/a83af7d89a154178bfdf255b841dc8dd, entries=200, sequenceid=63, filesize=38.6 K 2024-11-15T00:45:16,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2776a135c6b0473aaf31711e905f819b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2776a135c6b0473aaf31711e905f819b 2024-11-15T00:45:16,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2776a135c6b0473aaf31711e905f819b, entries=150, sequenceid=63, filesize=11.7 K 2024-11-15T00:45:16,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:16,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/14d1cd70f88c4ae79aaa1933b5936bf5 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/14d1cd70f88c4ae79aaa1933b5936bf5 2024-11-15T00:45:16,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:16,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:16,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/14d1cd70f88c4ae79aaa1933b5936bf5, entries=150, sequenceid=63, filesize=11.7 K 2024-11-15T00:45:16,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f762865d4d0d37fcd928550c43e0b373 in 1604ms, sequenceid=63, compaction requested=true 2024-11-15T00:45:16,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:16,942 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:16,942 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:16,943 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:16,943 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:16,943 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/B is initiating minor compaction (all files) 2024-11-15T00:45:16,943 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/A is initiating minor compaction (all files) 2024-11-15T00:45:16,943 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/B in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,943 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/A in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,943 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2d5eaa105f064bfeabba99c59515d573, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/10519e3d8bce4b279fa6ca2f1a08a64a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2776a135c6b0473aaf31711e905f819b] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=35.2 K 2024-11-15T00:45:16,943 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/90544f73eb2e4b0aa3300c8436a958ec, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/7fe24db973df4c8a82cc4233ca6e52bb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/a83af7d89a154178bfdf255b841dc8dd] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=99.1 K 2024-11-15T00:45:16,944 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d5eaa105f064bfeabba99c59515d573, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631514129 2024-11-15T00:45:16,943 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:16,944 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/90544f73eb2e4b0aa3300c8436a958ec, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/7fe24db973df4c8a82cc4233ca6e52bb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/a83af7d89a154178bfdf255b841dc8dd] 2024-11-15T00:45:16,944 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 10519e3d8bce4b279fa6ca2f1a08a64a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731631514315 2024-11-15T00:45:16,944 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90544f73eb2e4b0aa3300c8436a958ec, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631514129 2024-11-15T00:45:16,944 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2776a135c6b0473aaf31711e905f819b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631514961 2024-11-15T00:45:16,945 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe24db973df4c8a82cc4233ca6e52bb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731631514315 2024-11-15T00:45:16,945 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting a83af7d89a154178bfdf255b841dc8dd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631514961 2024-11-15T00:45:16,972 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:16,973 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/ebaac84af0bb498e8b9987c33c8f98b7 is 50, key is test_row_0/B:col10/1731631515335/Put/seqid=0 2024-11-15T00:45:16,973 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:17,009 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411152731427a1741436daa739689ea065301_f762865d4d0d37fcd928550c43e0b373 store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:17,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741939_1115 (size=12104) 2024-11-15T00:45:17,029 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411152731427a1741436daa739689ea065301_f762865d4d0d37fcd928550c43e0b373, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:17,029 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152731427a1741436daa739689ea065301_f762865d4d0d37fcd928550c43e0b373 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:17,042 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/ebaac84af0bb498e8b9987c33c8f98b7 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ebaac84af0bb498e8b9987c33c8f98b7 2024-11-15T00:45:17,049 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/B of f762865d4d0d37fcd928550c43e0b373 into ebaac84af0bb498e8b9987c33c8f98b7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:17,049 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:17,049 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/B, priority=13, startTime=1731631516942; duration=0sec 2024-11-15T00:45:17,050 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:17,050 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:B 2024-11-15T00:45:17,050 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:17,051 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:17,051 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/C is initiating minor compaction (all files) 2024-11-15T00:45:17,051 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/C in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:17,051 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/6b41657ed28f415fa13a8d86538f2aac, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/de29f95e193842e295c6e7bae36d298b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/14d1cd70f88c4ae79aaa1933b5936bf5] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=35.2 K 2024-11-15T00:45:17,052 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b41657ed28f415fa13a8d86538f2aac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631514129 2024-11-15T00:45:17,052 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting de29f95e193842e295c6e7bae36d298b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731631514315 2024-11-15T00:45:17,053 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d1cd70f88c4ae79aaa1933b5936bf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631514961 2024-11-15T00:45:17,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741940_1116 (size=4469) 2024-11-15T00:45:17,084 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:17,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T00:45:17,084 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/e0b389200e7d4f9da320ae3c8c877dde is 50, key is test_row_0/C:col10/1731631515335/Put/seqid=0 2024-11-15T00:45:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:17,085 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-15T00:45:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:17,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:17,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741941_1117 (size=12104) 2024-11-15T00:45:17,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f47519a4a6043a6b4a837f4b3c29085_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631515532/Put/seqid=0 2024-11-15T00:45:17,176 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/e0b389200e7d4f9da320ae3c8c877dde as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/e0b389200e7d4f9da320ae3c8c877dde 2024-11-15T00:45:17,184 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/C of f762865d4d0d37fcd928550c43e0b373 into e0b389200e7d4f9da320ae3c8c877dde(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:17,184 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:17,184 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/C, priority=13, startTime=1731631516942; duration=0sec 2024-11-15T00:45:17,185 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:17,185 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:C 2024-11-15T00:45:17,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741942_1118 (size=12154) 2024-11-15T00:45:17,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:17,198 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f47519a4a6043a6b4a837f4b3c29085_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f47519a4a6043a6b4a837f4b3c29085_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:17,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/6f70eed574b0433a8b614221be92d319, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:17,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/6f70eed574b0433a8b614221be92d319 is 175, key is test_row_0/A:col10/1731631515532/Put/seqid=0 2024-11-15T00:45:17,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741943_1119 (size=30955) 2024-11-15T00:45:17,292 INFO [master/fb2f6cb7846a:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T00:45:17,292 INFO [master/fb2f6cb7846a:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T00:45:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T00:45:17,478 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#A#compaction#97 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:17,479 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/e8065565998248929934e71d628d5a5b is 175, key is test_row_0/A:col10/1731631515335/Put/seqid=0 2024-11-15T00:45:17,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741944_1120 (size=31058) 2024-11-15T00:45:17,647 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/6f70eed574b0433a8b614221be92d319 2024-11-15T00:45:17,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:17,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/5e18d93e07494d60af708f29b22aca84 is 50, key is test_row_0/B:col10/1731631515532/Put/seqid=0 2024-11-15T00:45:17,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741945_1121 (size=12001) 2024-11-15T00:45:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631577711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:17,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631577736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:17,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631577739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:17,747 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:17,748 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:17,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631577749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:17,758 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:17,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631577760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:17,766 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:17,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:17,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631577819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:17,923 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/e8065565998248929934e71d628d5a5b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e8065565998248929934e71d628d5a5b 2024-11-15T00:45:17,930 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/A of f762865d4d0d37fcd928550c43e0b373 into e8065565998248929934e71d628d5a5b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:17,930 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:17,930 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/A, priority=13, startTime=1731631516941; duration=0sec 2024-11-15T00:45:17,930 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:17,930 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:A 2024-11-15T00:45:18,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631578029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:18,105 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/5e18d93e07494d60af708f29b22aca84 2024-11-15T00:45:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/603afdb7733a4beaae60add27c09da8c is 50, key is test_row_0/C:col10/1731631515532/Put/seqid=0 2024-11-15T00:45:18,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741946_1122 (size=12001) 2024-11-15T00:45:18,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631578349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:18,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:18,633 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/603afdb7733a4beaae60add27c09da8c 2024-11-15T00:45:18,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/6f70eed574b0433a8b614221be92d319 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/6f70eed574b0433a8b614221be92d319 2024-11-15T00:45:18,659 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/6f70eed574b0433a8b614221be92d319, entries=150, sequenceid=98, filesize=30.2 K 2024-11-15T00:45:18,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/5e18d93e07494d60af708f29b22aca84 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5e18d93e07494d60af708f29b22aca84 2024-11-15T00:45:18,670 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5e18d93e07494d60af708f29b22aca84, entries=150, sequenceid=98, filesize=11.7 K 2024-11-15T00:45:18,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/603afdb7733a4beaae60add27c09da8c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/603afdb7733a4beaae60add27c09da8c 2024-11-15T00:45:18,687 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/603afdb7733a4beaae60add27c09da8c, entries=150, sequenceid=98, filesize=11.7 K 2024-11-15T00:45:18,697 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for f762865d4d0d37fcd928550c43e0b373 in 1612ms, sequenceid=98, compaction requested=false 2024-11-15T00:45:18,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:18,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:18,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-15T00:45:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-15T00:45:18,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-15T00:45:18,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5030 sec 2024-11-15T00:45:18,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.5180 sec 2024-11-15T00:45:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:18,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:45:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:18,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152e211dfa01d14c44b931aa63dc41359f_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631517703/Put/seqid=0 2024-11-15T00:45:18,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741947_1123 (size=14594) 2024-11-15T00:45:18,962 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:18,969 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152e211dfa01d14c44b931aa63dc41359f_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152e211dfa01d14c44b931aa63dc41359f_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:18,974 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/0b194f4f12874314ba80e44f4fee8dbc, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:18,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/0b194f4f12874314ba80e44f4fee8dbc is 175, key is test_row_0/A:col10/1731631517703/Put/seqid=0 2024-11-15T00:45:19,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741948_1124 (size=39549) 2024-11-15T00:45:19,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631579105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:19,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631579230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T00:45:19,400 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:19,401 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T00:45:19,403 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:45:19,404 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:19,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:19,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/0b194f4f12874314ba80e44f4fee8dbc 2024-11-15T00:45:19,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631579443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:19,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2bfcab08730d4fcb919a84acb2a58233 is 50, key is test_row_0/B:col10/1731631517703/Put/seqid=0 2024-11-15T00:45:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:45:19,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741949_1125 (size=12001) 2024-11-15T00:45:19,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2bfcab08730d4fcb919a84acb2a58233 2024-11-15T00:45:19,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/9011e09bd9c348319f50a5417a8d7d08 is 50, key is test_row_0/C:col10/1731631517703/Put/seqid=0 2024-11-15T00:45:19,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:45:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:19,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:19,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:19,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:19,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741950_1126 (size=12001) 2024-11-15T00:45:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:45:19,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:45:19,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:19,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:19,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:19,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:19,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:19,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631579759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:19,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631579759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:19,763 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:19,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:19,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:19,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631579769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:19,773 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:19,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:19,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631579780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:19,784 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:19,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631579789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:19,790 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:19,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:45:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:19,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:19,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:19,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/9011e09bd9c348319f50a5417a8d7d08 2024-11-15T00:45:20,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/0b194f4f12874314ba80e44f4fee8dbc as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/0b194f4f12874314ba80e44f4fee8dbc 2024-11-15T00:45:20,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/0b194f4f12874314ba80e44f4fee8dbc, entries=200, sequenceid=112, filesize=38.6 K 2024-11-15T00:45:20,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2bfcab08730d4fcb919a84acb2a58233 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2bfcab08730d4fcb919a84acb2a58233 2024-11-15T00:45:20,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2bfcab08730d4fcb919a84acb2a58233, entries=150, sequenceid=112, filesize=11.7 K 2024-11-15T00:45:20,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/9011e09bd9c348319f50a5417a8d7d08 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/9011e09bd9c348319f50a5417a8d7d08 2024-11-15T00:45:20,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:45:20,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:20,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:20,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:20,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:20,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:45:20,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/9011e09bd9c348319f50a5417a8d7d08, entries=150, sequenceid=112, filesize=11.7 K 2024-11-15T00:45:20,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f762865d4d0d37fcd928550c43e0b373 in 1154ms, sequenceid=112, compaction requested=true 2024-11-15T00:45:20,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:20,031 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:20,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:20,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:20,033 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:20,033 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:20,033 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/A is initiating minor compaction (all files) 2024-11-15T00:45:20,033 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/A in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:20,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:20,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:20,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:20,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:20,033 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e8065565998248929934e71d628d5a5b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/6f70eed574b0433a8b614221be92d319, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/0b194f4f12874314ba80e44f4fee8dbc] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=99.2 K 2024-11-15T00:45:20,033 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:20,037 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:20,038 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/B is initiating minor compaction (all files) 2024-11-15T00:45:20,038 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/B in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:20,038 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ebaac84af0bb498e8b9987c33c8f98b7, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5e18d93e07494d60af708f29b22aca84, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2bfcab08730d4fcb919a84acb2a58233] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=35.3 K 2024-11-15T00:45:20,033 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e8065565998248929934e71d628d5a5b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/6f70eed574b0433a8b614221be92d319, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/0b194f4f12874314ba80e44f4fee8dbc] 2024-11-15T00:45:20,038 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ebaac84af0bb498e8b9987c33c8f98b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631514961 2024-11-15T00:45:20,039 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8065565998248929934e71d628d5a5b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631514961 2024-11-15T00:45:20,039 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e18d93e07494d60af708f29b22aca84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731631515532 2024-11-15T00:45:20,039 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f70eed574b0433a8b614221be92d319, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731631515532 2024-11-15T00:45:20,040 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bfcab08730d4fcb919a84acb2a58233, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731631517703 2024-11-15T00:45:20,040 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b194f4f12874314ba80e44f4fee8dbc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731631517696 2024-11-15T00:45:20,057 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:20,059 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:20,059 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/d42b76d3d66042bc8988a74ff91a41db is 50, key is test_row_0/B:col10/1731631517703/Put/seqid=0 2024-11-15T00:45:20,061 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115d4318ca96dc341b09e05eb5698101e97_f762865d4d0d37fcd928550c43e0b373 store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:20,065 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115d4318ca96dc341b09e05eb5698101e97_f762865d4d0d37fcd928550c43e0b373, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:20,065 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d4318ca96dc341b09e05eb5698101e97_f762865d4d0d37fcd928550c43e0b373 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:20,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741951_1127 (size=12207) 2024-11-15T00:45:20,086 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/d42b76d3d66042bc8988a74ff91a41db as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/d42b76d3d66042bc8988a74ff91a41db 2024-11-15T00:45:20,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741952_1128 (size=4469) 2024-11-15T00:45:20,099 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/B of f762865d4d0d37fcd928550c43e0b373 into d42b76d3d66042bc8988a74ff91a41db(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:20,099 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:20,099 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/B, priority=13, startTime=1731631520033; duration=0sec 2024-11-15T00:45:20,099 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:20,099 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:B 2024-11-15T00:45:20,099 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:20,101 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:20,101 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/C is initiating minor compaction (all files) 2024-11-15T00:45:20,101 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/C in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:20,102 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/e0b389200e7d4f9da320ae3c8c877dde, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/603afdb7733a4beaae60add27c09da8c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/9011e09bd9c348319f50a5417a8d7d08] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=35.3 K 2024-11-15T00:45:20,105 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#A#compaction#106 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:20,106 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/e9f66da13f4e466f9174db94a5ad0b23 is 175, key is test_row_0/A:col10/1731631517703/Put/seqid=0 2024-11-15T00:45:20,110 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e0b389200e7d4f9da320ae3c8c877dde, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631514961 2024-11-15T00:45:20,110 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 603afdb7733a4beaae60add27c09da8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731631515532 2024-11-15T00:45:20,111 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 9011e09bd9c348319f50a5417a8d7d08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731631517703 2024-11-15T00:45:20,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741953_1129 (size=31161) 2024-11-15T00:45:20,124 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:20,125 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/d758e77250274bb9811e011c15fe5ec7 is 50, key is test_row_0/C:col10/1731631517703/Put/seqid=0 2024-11-15T00:45:20,137 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/e9f66da13f4e466f9174db94a5ad0b23 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e9f66da13f4e466f9174db94a5ad0b23 2024-11-15T00:45:20,160 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/A of f762865d4d0d37fcd928550c43e0b373 into e9f66da13f4e466f9174db94a5ad0b23(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:20,160 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:20,160 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/A, priority=13, startTime=1731631520031; duration=0sec 2024-11-15T00:45:20,160 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:20,160 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:A 2024-11-15T00:45:20,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741954_1130 (size=12207) 2024-11-15T00:45:20,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:45:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:20,183 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:45:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:20,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:20,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111572c1c2291761423cbb7a3e485e230f07_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631519096/Put/seqid=0 2024-11-15T00:45:20,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741955_1131 (size=12254) 2024-11-15T00:45:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:20,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:20,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631580357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:20,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631580469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:45:20,601 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/d758e77250274bb9811e011c15fe5ec7 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/d758e77250274bb9811e011c15fe5ec7 2024-11-15T00:45:20,629 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/C of f762865d4d0d37fcd928550c43e0b373 into d758e77250274bb9811e011c15fe5ec7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:20,629 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:20,629 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/C, priority=13, startTime=1731631520033; duration=0sec 2024-11-15T00:45:20,629 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:20,630 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:C 2024-11-15T00:45:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:20,671 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111572c1c2291761423cbb7a3e485e230f07_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111572c1c2291761423cbb7a3e485e230f07_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:20,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/24e614015e914e25ad8bd18301043531, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:20,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/24e614015e914e25ad8bd18301043531 is 175, key is test_row_0/A:col10/1731631519096/Put/seqid=0 2024-11-15T00:45:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631580692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:20,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741956_1132 (size=31055) 2024-11-15T00:45:21,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631580999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:21,100 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/24e614015e914e25ad8bd18301043531 2024-11-15T00:45:21,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/444c60080c1b499291ad9d28078631b9 is 50, key is test_row_0/B:col10/1731631519096/Put/seqid=0 2024-11-15T00:45:21,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741957_1133 (size=12101) 2024-11-15T00:45:21,176 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/444c60080c1b499291ad9d28078631b9 2024-11-15T00:45:21,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/676dd97850714a2db611606e4bae84b0 is 50, key is test_row_0/C:col10/1731631519096/Put/seqid=0 2024-11-15T00:45:21,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741958_1134 (size=12101) 2024-11-15T00:45:21,259 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/676dd97850714a2db611606e4bae84b0 2024-11-15T00:45:21,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/24e614015e914e25ad8bd18301043531 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/24e614015e914e25ad8bd18301043531 2024-11-15T00:45:21,284 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/24e614015e914e25ad8bd18301043531, entries=150, sequenceid=134, filesize=30.3 K 2024-11-15T00:45:21,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/444c60080c1b499291ad9d28078631b9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/444c60080c1b499291ad9d28078631b9 2024-11-15T00:45:21,301 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/444c60080c1b499291ad9d28078631b9, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T00:45:21,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/676dd97850714a2db611606e4bae84b0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/676dd97850714a2db611606e4bae84b0 2024-11-15T00:45:21,312 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/676dd97850714a2db611606e4bae84b0, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T00:45:21,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f762865d4d0d37fcd928550c43e0b373 in 1130ms, sequenceid=134, compaction requested=false 2024-11-15T00:45:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T00:45:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T00:45:21,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T00:45:21,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9110 sec 2024-11-15T00:45:21,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9180 sec 2024-11-15T00:45:21,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:45:21,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:21,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:21,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:21,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:21,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:21,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:45:21,568 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:21,572 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:21,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b6722f89d910488b9c41884d7f1a3d67_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631520341/Put/seqid=0 2024-11-15T00:45:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T00:45:21,575 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:21,577 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:21,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:45:21,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741959_1135 (size=14794) 2024-11-15T00:45:21,629 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:21,650 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b6722f89d910488b9c41884d7f1a3d67_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b6722f89d910488b9c41884d7f1a3d67_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:21,657 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/ffaa0d74a40848368c6cc324f600caf9, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:21,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/ffaa0d74a40848368c6cc324f600caf9 is 175, key is test_row_0/A:col10/1731631520341/Put/seqid=0 2024-11-15T00:45:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:45:21,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741960_1136 (size=39749) 2024-11-15T00:45:21,707 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/ffaa0d74a40848368c6cc324f600caf9 2024-11-15T00:45:21,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/610f0f3bf18e426d93c314ff4645d561 is 50, key is test_row_0/B:col10/1731631520341/Put/seqid=0 2024-11-15T00:45:21,742 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T00:45:21,742 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T00:45:21,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T00:45:21,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:21,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:21,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:21,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:21,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:21,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741961_1137 (size=12151) 2024-11-15T00:45:21,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631581830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:45:21,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T00:45:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:21,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:22,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631581995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:22,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T00:45:22,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:22,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:22,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:22,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:22,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:22,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/610f0f3bf18e426d93c314ff4645d561 2024-11-15T00:45:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:45:22,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T00:45:22,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631582212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:22,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:22,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/aafa51c8a9ae41ec8c20cc93f7885a2d is 50, key is test_row_0/C:col10/1731631520341/Put/seqid=0 2024-11-15T00:45:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:22,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741962_1138 (size=12151) 2024-11-15T00:45:22,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/aafa51c8a9ae41ec8c20cc93f7885a2d 2024-11-15T00:45:22,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/ffaa0d74a40848368c6cc324f600caf9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/ffaa0d74a40848368c6cc324f600caf9 2024-11-15T00:45:22,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/ffaa0d74a40848368c6cc324f600caf9, entries=200, sequenceid=148, filesize=38.8 K 2024-11-15T00:45:22,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/610f0f3bf18e426d93c314ff4645d561 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/610f0f3bf18e426d93c314ff4645d561 2024-11-15T00:45:22,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/610f0f3bf18e426d93c314ff4645d561, entries=150, sequenceid=148, filesize=11.9 K 2024-11-15T00:45:22,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/aafa51c8a9ae41ec8c20cc93f7885a2d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/aafa51c8a9ae41ec8c20cc93f7885a2d 2024-11-15T00:45:22,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/aafa51c8a9ae41ec8c20cc93f7885a2d, entries=150, sequenceid=148, filesize=11.9 K 2024-11-15T00:45:22,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f762865d4d0d37fcd928550c43e0b373 in 834ms, sequenceid=148, compaction requested=true 2024-11-15T00:45:22,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:22,366 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:22,367 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:22,367 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/A is initiating minor compaction (all files) 2024-11-15T00:45:22,367 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/A in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:22,368 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e9f66da13f4e466f9174db94a5ad0b23, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/24e614015e914e25ad8bd18301043531, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/ffaa0d74a40848368c6cc324f600caf9] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=99.6 K 2024-11-15T00:45:22,368 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:22,368 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e9f66da13f4e466f9174db94a5ad0b23, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/24e614015e914e25ad8bd18301043531, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/ffaa0d74a40848368c6cc324f600caf9] 2024-11-15T00:45:22,369 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9f66da13f4e466f9174db94a5ad0b23, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731631517703 2024-11-15T00:45:22,369 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e614015e914e25ad8bd18301043531, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631519010 2024-11-15T00:45:22,369 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffaa0d74a40848368c6cc324f600caf9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731631520320 2024-11-15T00:45:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:22,384 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:22,385 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:22,385 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/B is initiating minor compaction (all files) 2024-11-15T00:45:22,385 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/B in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:22,385 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/d42b76d3d66042bc8988a74ff91a41db, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/444c60080c1b499291ad9d28078631b9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/610f0f3bf18e426d93c314ff4645d561] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=35.6 K 2024-11-15T00:45:22,386 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting d42b76d3d66042bc8988a74ff91a41db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731631517703 2024-11-15T00:45:22,386 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 444c60080c1b499291ad9d28078631b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631519010 2024-11-15T00:45:22,386 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 610f0f3bf18e426d93c314ff4645d561, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731631520320 2024-11-15T00:45:22,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:22,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:22,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:22,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:22,397 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:22,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T00:45:22,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:22,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T00:45:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:22,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:22,418 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:22,419 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/bd83c877ec1c433da174670446302512 is 50, key is test_row_0/B:col10/1731631520341/Put/seqid=0 2024-11-15T00:45:22,421 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111538dd64c176fe4bbd85210dc2690436ff_f762865d4d0d37fcd928550c43e0b373 store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:22,424 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111538dd64c176fe4bbd85210dc2690436ff_f762865d4d0d37fcd928550c43e0b373, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:22,426 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111538dd64c176fe4bbd85210dc2690436ff_f762865d4d0d37fcd928550c43e0b373 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:22,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741963_1139 (size=12459) 2024-11-15T00:45:22,498 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/bd83c877ec1c433da174670446302512 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/bd83c877ec1c433da174670446302512 2024-11-15T00:45:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115184c07f1a16f4ff2b65f96bd42f0599c_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631521819/Put/seqid=0 2024-11-15T00:45:22,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741964_1140 (size=4469) 2024-11-15T00:45:22,511 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/B of f762865d4d0d37fcd928550c43e0b373 into bd83c877ec1c433da174670446302512(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:22,511 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:22,511 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/B, priority=13, startTime=1731631522383; duration=0sec 2024-11-15T00:45:22,511 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:22,511 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:B 2024-11-15T00:45:22,511 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:22,513 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:22,514 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/C is initiating minor compaction (all files) 2024-11-15T00:45:22,514 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/C in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:22,514 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/d758e77250274bb9811e011c15fe5ec7, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/676dd97850714a2db611606e4bae84b0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/aafa51c8a9ae41ec8c20cc93f7885a2d] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=35.6 K 2024-11-15T00:45:22,515 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting d758e77250274bb9811e011c15fe5ec7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731631517703 2024-11-15T00:45:22,515 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 676dd97850714a2db611606e4bae84b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631519010 2024-11-15T00:45:22,516 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting aafa51c8a9ae41ec8c20cc93f7885a2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731631520320 2024-11-15T00:45:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:22,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:22,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741965_1141 (size=12304) 2024-11-15T00:45:22,570 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:22,571 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/3d48142371674e1d9b78de4577dce511 is 50, key is test_row_0/C:col10/1731631520341/Put/seqid=0 2024-11-15T00:45:22,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741966_1142 (size=12459) 2024-11-15T00:45:22,662 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/3d48142371674e1d9b78de4577dce511 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3d48142371674e1d9b78de4577dce511 2024-11-15T00:45:22,683 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/C of f762865d4d0d37fcd928550c43e0b373 into 3d48142371674e1d9b78de4577dce511(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:22,683 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:22,683 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/C, priority=13, startTime=1731631522396; duration=0sec 2024-11-15T00:45:22,683 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:22,683 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:C 2024-11-15T00:45:22,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631582686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:45:22,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631582799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:22,908 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#A#compaction#114 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:22,909 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/be50547b154041ceba96a514f3d1f96c is 175, key is test_row_0/A:col10/1731631520341/Put/seqid=0 2024-11-15T00:45:22,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741967_1143 (size=31413) 2024-11-15T00:45:22,955 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/be50547b154041ceba96a514f3d1f96c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/be50547b154041ceba96a514f3d1f96c 2024-11-15T00:45:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:22,974 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115184c07f1a16f4ff2b65f96bd42f0599c_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115184c07f1a16f4ff2b65f96bd42f0599c_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:22,974 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/A of f762865d4d0d37fcd928550c43e0b373 into be50547b154041ceba96a514f3d1f96c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:22,974 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:22,974 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/A, priority=13, startTime=1731631522366; duration=0sec 2024-11-15T00:45:22,975 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:22,975 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:A 2024-11-15T00:45:22,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/255adccb999641eb9074e6cc342cb25a, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:22,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/255adccb999641eb9074e6cc342cb25a is 175, key is test_row_0/A:col10/1731631521819/Put/seqid=0 2024-11-15T00:45:23,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631583019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:23,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741968_1144 (size=31105) 2024-11-15T00:45:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631583334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:23,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:23,460 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/255adccb999641eb9074e6cc342cb25a 2024-11-15T00:45:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/ddf754736d844768b1e6032afd0d72d9 is 50, key is test_row_0/B:col10/1731631521819/Put/seqid=0 2024-11-15T00:45:23,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741969_1145 (size=12151) 2024-11-15T00:45:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:45:23,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631583790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:23,803 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:23,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631583796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:23,808 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:23,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631583800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:23,824 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:23,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631583832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:23,835 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:23,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:23,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631583850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:24,025 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/ddf754736d844768b1e6032afd0d72d9 2024-11-15T00:45:24,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/8332d3a26e11420eaba47f005c170db6 is 50, key is test_row_0/C:col10/1731631521819/Put/seqid=0 2024-11-15T00:45:24,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741970_1146 (size=12151) 2024-11-15T00:45:24,169 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/8332d3a26e11420eaba47f005c170db6 2024-11-15T00:45:24,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/255adccb999641eb9074e6cc342cb25a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/255adccb999641eb9074e6cc342cb25a 2024-11-15T00:45:24,196 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/255adccb999641eb9074e6cc342cb25a, entries=150, sequenceid=167, filesize=30.4 K 2024-11-15T00:45:24,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/ddf754736d844768b1e6032afd0d72d9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ddf754736d844768b1e6032afd0d72d9 2024-11-15T00:45:24,217 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ddf754736d844768b1e6032afd0d72d9, entries=150, sequenceid=167, filesize=11.9 K 2024-11-15T00:45:24,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/8332d3a26e11420eaba47f005c170db6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8332d3a26e11420eaba47f005c170db6 2024-11-15T00:45:24,229 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8332d3a26e11420eaba47f005c170db6, entries=150, sequenceid=167, filesize=11.9 K 2024-11-15T00:45:24,231 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f762865d4d0d37fcd928550c43e0b373 in 1826ms, sequenceid=167, compaction requested=false 2024-11-15T00:45:24,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:24,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:24,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T00:45:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T00:45:24,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T00:45:24,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6600 sec 2024-11-15T00:45:24,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.6710 sec 2024-11-15T00:45:24,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T00:45:24,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:24,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:24,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:24,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:24,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:24,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:24,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111579d4b99af67643babb62b6a75b095d50_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631522683/Put/seqid=0 2024-11-15T00:45:24,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741971_1147 (size=14794) 2024-11-15T00:45:24,967 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:24,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111579d4b99af67643babb62b6a75b095d50_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111579d4b99af67643babb62b6a75b095d50_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:24,984 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/4917526c3148468e89214bd1610e4cd8, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:24,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/4917526c3148468e89214bd1610e4cd8 is 175, key is test_row_0/A:col10/1731631522683/Put/seqid=0 2024-11-15T00:45:25,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741972_1148 (size=39749) 2024-11-15T00:45:25,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631585118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:25,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:25,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:25,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631585229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:25,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/4917526c3148468e89214bd1610e4cd8 2024-11-15T00:45:25,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631585439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:25,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/986a442dc00242d9bc52b7581ee73aa2 is 50, key is test_row_0/B:col10/1731631522683/Put/seqid=0 2024-11-15T00:45:25,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741973_1149 (size=12151) 2024-11-15T00:45:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:45:25,760 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:25,760 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T00:45:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:45:25,764 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:25,765 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:25,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:25,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631585759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:45:25,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/986a442dc00242d9bc52b7581ee73aa2 2024-11-15T00:45:25,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T00:45:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:25,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:25,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/52ef6f7f4685488cbcfaecd6bbb67cfd is 50, key is test_row_0/C:col10/1731631522683/Put/seqid=0 2024-11-15T00:45:25,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741974_1150 (size=12151) 2024-11-15T00:45:25,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/52ef6f7f4685488cbcfaecd6bbb67cfd 2024-11-15T00:45:25,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/4917526c3148468e89214bd1610e4cd8 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/4917526c3148468e89214bd1610e4cd8 2024-11-15T00:45:25,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/4917526c3148468e89214bd1610e4cd8, entries=200, sequenceid=184, filesize=38.8 K 2024-11-15T00:45:25,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/986a442dc00242d9bc52b7581ee73aa2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/986a442dc00242d9bc52b7581ee73aa2 2024-11-15T00:45:26,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/986a442dc00242d9bc52b7581ee73aa2, entries=150, sequenceid=184, filesize=11.9 K 2024-11-15T00:45:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/52ef6f7f4685488cbcfaecd6bbb67cfd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/52ef6f7f4685488cbcfaecd6bbb67cfd 2024-11-15T00:45:26,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/52ef6f7f4685488cbcfaecd6bbb67cfd, entries=150, sequenceid=184, filesize=11.9 K 2024-11-15T00:45:26,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f762865d4d0d37fcd928550c43e0b373 in 1134ms, sequenceid=184, compaction requested=true 2024-11-15T00:45:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:26,031 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:26,031 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:26,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:26,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:26,032 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:26,032 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:26,033 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/B is initiating minor compaction (all files) 2024-11-15T00:45:26,033 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/A is initiating minor compaction (all files) 2024-11-15T00:45:26,033 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/B in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:26,033 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/A in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:26,033 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/bd83c877ec1c433da174670446302512, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ddf754736d844768b1e6032afd0d72d9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/986a442dc00242d9bc52b7581ee73aa2] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=35.9 K 2024-11-15T00:45:26,033 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/be50547b154041ceba96a514f3d1f96c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/255adccb999641eb9074e6cc342cb25a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/4917526c3148468e89214bd1610e4cd8] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=99.9 K 2024-11-15T00:45:26,033 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:26,033 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/be50547b154041ceba96a514f3d1f96c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/255adccb999641eb9074e6cc342cb25a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/4917526c3148468e89214bd1610e4cd8] 2024-11-15T00:45:26,034 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting bd83c877ec1c433da174670446302512, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731631520320 2024-11-15T00:45:26,037 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting be50547b154041ceba96a514f3d1f96c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731631520320 2024-11-15T00:45:26,037 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 255adccb999641eb9074e6cc342cb25a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731631521803 2024-11-15T00:45:26,037 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf754736d844768b1e6032afd0d72d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731631521803 2024-11-15T00:45:26,038 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4917526c3148468e89214bd1610e4cd8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631522592 2024-11-15T00:45:26,038 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 986a442dc00242d9bc52b7581ee73aa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631522592 2024-11-15T00:45:26,049 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:26,053 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:26,053 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/c1a6b480a3ea446285083558ee4db781 is 50, key is test_row_0/B:col10/1731631522683/Put/seqid=0 2024-11-15T00:45:26,069 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115535315f430254e9380761b956d18cae1_f762865d4d0d37fcd928550c43e0b373 store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:26,072 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115535315f430254e9380761b956d18cae1_f762865d4d0d37fcd928550c43e0b373, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:26,072 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115535315f430254e9380761b956d18cae1_f762865d4d0d37fcd928550c43e0b373 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:26,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T00:45:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:26,082 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T00:45:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:45:26,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741975_1151 (size=12561) 2024-11-15T00:45:26,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741976_1152 (size=4469) 2024-11-15T00:45:26,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111569d356a0bbae485893c94b42fc5db9a2_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631525116/Put/seqid=0 2024-11-15T00:45:26,208 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#A#compaction#123 average throughput is 0.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:26,209 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/c4bcbe2bf3b9496789fe7022b38da9d1 is 175, key is test_row_0/A:col10/1731631522683/Put/seqid=0 2024-11-15T00:45:26,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741977_1153 (size=12304) 2024-11-15T00:45:26,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741978_1154 (size=31515) 2024-11-15T00:45:26,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:26,296 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/c4bcbe2bf3b9496789fe7022b38da9d1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/c4bcbe2bf3b9496789fe7022b38da9d1 2024-11-15T00:45:26,314 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/A of f762865d4d0d37fcd928550c43e0b373 into c4bcbe2bf3b9496789fe7022b38da9d1(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:26,314 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:26,314 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/A, priority=13, startTime=1731631526031; duration=0sec 2024-11-15T00:45:26,314 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:26,314 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:A 2024-11-15T00:45:26,315 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:26,316 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:26,316 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/C is initiating minor compaction (all files) 2024-11-15T00:45:26,316 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/C in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:26,316 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3d48142371674e1d9b78de4577dce511, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8332d3a26e11420eaba47f005c170db6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/52ef6f7f4685488cbcfaecd6bbb67cfd] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=35.9 K 2024-11-15T00:45:26,319 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d48142371674e1d9b78de4577dce511, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731631520320 2024-11-15T00:45:26,320 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8332d3a26e11420eaba47f005c170db6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731631521803 2024-11-15T00:45:26,322 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ef6f7f4685488cbcfaecd6bbb67cfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631522592 2024-11-15T00:45:26,339 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:26,340 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/4bf24f3cea314b60aad4aad7d6e4c88a is 50, key is test_row_0/C:col10/1731631522683/Put/seqid=0 2024-11-15T00:45:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:45:26,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741979_1155 (size=12561) 2024-11-15T00:45:26,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631586439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:26,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:26,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:26,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631586550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:26,582 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/c1a6b480a3ea446285083558ee4db781 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/c1a6b480a3ea446285083558ee4db781 2024-11-15T00:45:26,597 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/B of f762865d4d0d37fcd928550c43e0b373 into c1a6b480a3ea446285083558ee4db781(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:26,597 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:26,597 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/B, priority=13, startTime=1731631526031; duration=0sec 2024-11-15T00:45:26,597 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:26,597 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:B 2024-11-15T00:45:26,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:26,666 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111569d356a0bbae485893c94b42fc5db9a2_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569d356a0bbae485893c94b42fc5db9a2_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:26,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/16628b82ac154e3ea0bf0025a697b67a, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:26,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/16628b82ac154e3ea0bf0025a697b67a is 175, key is test_row_0/A:col10/1731631525116/Put/seqid=0 2024-11-15T00:45:26,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741980_1156 (size=31105) 2024-11-15T00:45:26,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631586759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:26,807 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/4bf24f3cea314b60aad4aad7d6e4c88a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/4bf24f3cea314b60aad4aad7d6e4c88a 2024-11-15T00:45:26,820 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/C of f762865d4d0d37fcd928550c43e0b373 into 4bf24f3cea314b60aad4aad7d6e4c88a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:26,820 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:26,820 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/C, priority=13, startTime=1731631526031; duration=0sec 2024-11-15T00:45:26,820 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:26,820 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:C 2024-11-15T00:45:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:45:27,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631587070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:27,173 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/16628b82ac154e3ea0bf0025a697b67a 2024-11-15T00:45:27,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/f2ffcde84467462880d522c0c764a2df is 50, key is test_row_0/B:col10/1731631525116/Put/seqid=0 2024-11-15T00:45:27,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741981_1157 (size=12151) 2024-11-15T00:45:27,252 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/f2ffcde84467462880d522c0c764a2df 2024-11-15T00:45:27,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/56f9cc5fa524496bb9203fb0c9f96cbd is 50, key is test_row_0/C:col10/1731631525116/Put/seqid=0 2024-11-15T00:45:27,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741982_1158 (size=12151) 2024-11-15T00:45:27,312 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/56f9cc5fa524496bb9203fb0c9f96cbd 2024-11-15T00:45:27,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/16628b82ac154e3ea0bf0025a697b67a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/16628b82ac154e3ea0bf0025a697b67a 2024-11-15T00:45:27,347 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/16628b82ac154e3ea0bf0025a697b67a, entries=150, sequenceid=202, filesize=30.4 K 2024-11-15T00:45:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/f2ffcde84467462880d522c0c764a2df as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/f2ffcde84467462880d522c0c764a2df 2024-11-15T00:45:27,370 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/f2ffcde84467462880d522c0c764a2df, entries=150, sequenceid=202, filesize=11.9 K 2024-11-15T00:45:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/56f9cc5fa524496bb9203fb0c9f96cbd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/56f9cc5fa524496bb9203fb0c9f96cbd 2024-11-15T00:45:27,393 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/56f9cc5fa524496bb9203fb0c9f96cbd, entries=150, sequenceid=202, filesize=11.9 K 2024-11-15T00:45:27,395 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f762865d4d0d37fcd928550c43e0b373 in 1314ms, sequenceid=202, compaction requested=false 2024-11-15T00:45:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T00:45:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T00:45:27,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T00:45:27,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6330 sec 2024-11-15T00:45:27,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.6420 sec 2024-11-15T00:45:27,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T00:45:27,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:27,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:27,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:27,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:27,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:27,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:27,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111545221760f1b94fe8a748e7ffb6301f18_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631527579/Put/seqid=0 2024-11-15T00:45:27,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741983_1159 (size=14794) 2024-11-15T00:45:27,662 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:27,677 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111545221760f1b94fe8a748e7ffb6301f18_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111545221760f1b94fe8a748e7ffb6301f18_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:27,679 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/202cfb2c1e824894a873f63ee463218d, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:27,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/202cfb2c1e824894a873f63ee463218d is 175, key is test_row_0/A:col10/1731631527579/Put/seqid=0 2024-11-15T00:45:27,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741984_1160 (size=39749) 2024-11-15T00:45:27,704 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/202cfb2c1e824894a873f63ee463218d 2024-11-15T00:45:27,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631587750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:27,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/5cf10224bba6439a9495f3686c57bd36 is 50, key is test_row_0/B:col10/1731631527579/Put/seqid=0 2024-11-15T00:45:27,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741985_1161 (size=12151) 2024-11-15T00:45:27,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/5cf10224bba6439a9495f3686c57bd36 2024-11-15T00:45:27,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/3adda042f3f942bc95107dba0189d478 is 50, key is test_row_0/C:col10/1731631527579/Put/seqid=0 2024-11-15T00:45:27,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741986_1162 (size=12151) 2024-11-15T00:45:27,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631587872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:45:27,914 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:27,915 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T00:45:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:45:27,918 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:27,919 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:27,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:45:28,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T00:45:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:28,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:28,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631588090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:28,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:45:28,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T00:45:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:28,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:28,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/3adda042f3f942bc95107dba0189d478 2024-11-15T00:45:28,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/202cfb2c1e824894a873f63ee463218d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/202cfb2c1e824894a873f63ee463218d 2024-11-15T00:45:28,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/202cfb2c1e824894a873f63ee463218d, entries=200, sequenceid=220, filesize=38.8 K 2024-11-15T00:45:28,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/5cf10224bba6439a9495f3686c57bd36 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5cf10224bba6439a9495f3686c57bd36 2024-11-15T00:45:28,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5cf10224bba6439a9495f3686c57bd36, entries=150, sequenceid=220, filesize=11.9 K 2024-11-15T00:45:28,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/3adda042f3f942bc95107dba0189d478 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3adda042f3f942bc95107dba0189d478 2024-11-15T00:45:28,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3adda042f3f942bc95107dba0189d478, entries=150, sequenceid=220, filesize=11.9 K 2024-11-15T00:45:28,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f762865d4d0d37fcd928550c43e0b373 in 711ms, sequenceid=220, compaction requested=true 2024-11-15T00:45:28,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:28,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:28,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:28,317 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:28,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:28,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:28,317 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:28,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:28,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:28,318 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:28,318 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/B is initiating minor compaction (all files) 2024-11-15T00:45:28,319 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:28,319 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/A is initiating minor compaction (all files) 2024-11-15T00:45:28,319 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/A in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:28,319 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/c4bcbe2bf3b9496789fe7022b38da9d1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/16628b82ac154e3ea0bf0025a697b67a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/202cfb2c1e824894a873f63ee463218d] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=100.0 K 2024-11-15T00:45:28,319 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:28,319 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/c4bcbe2bf3b9496789fe7022b38da9d1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/16628b82ac154e3ea0bf0025a697b67a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/202cfb2c1e824894a873f63ee463218d] 2024-11-15T00:45:28,320 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/B in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:28,321 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/c1a6b480a3ea446285083558ee4db781, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/f2ffcde84467462880d522c0c764a2df, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5cf10224bba6439a9495f3686c57bd36] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=36.0 K 2024-11-15T00:45:28,324 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4bcbe2bf3b9496789fe7022b38da9d1, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631522592 2024-11-15T00:45:28,324 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a6b480a3ea446285083558ee4db781, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631522592 2024-11-15T00:45:28,324 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f2ffcde84467462880d522c0c764a2df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731631525056 2024-11-15T00:45:28,325 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16628b82ac154e3ea0bf0025a697b67a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731631525056 2024-11-15T00:45:28,325 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf10224bba6439a9495f3686c57bd36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731631526401 2024-11-15T00:45:28,325 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 202cfb2c1e824894a873f63ee463218d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731631526401 2024-11-15T00:45:28,349 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:28,361 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:28,362 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/586916ae814e49a595b200fbc72ef27b is 50, key is test_row_0/B:col10/1731631527579/Put/seqid=0 2024-11-15T00:45:28,364 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115354b73711cbe4ea987247c75e2887b5c_f762865d4d0d37fcd928550c43e0b373 store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:28,368 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115354b73711cbe4ea987247c75e2887b5c_f762865d4d0d37fcd928550c43e0b373, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:28,369 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115354b73711cbe4ea987247c75e2887b5c_f762865d4d0d37fcd928550c43e0b373 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:28,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741988_1164 (size=4469) 2024-11-15T00:45:28,384 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#A#compaction#132 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:28,384 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/b93d49bee54147c190994497ec689f3e is 175, key is test_row_0/A:col10/1731631527579/Put/seqid=0 2024-11-15T00:45:28,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741987_1163 (size=12663) 2024-11-15T00:45:28,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T00:45:28,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:28,403 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T00:45:28,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:28,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:28,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:28,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:28,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:28,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:28,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:28,409 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/586916ae814e49a595b200fbc72ef27b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/586916ae814e49a595b200fbc72ef27b 2024-11-15T00:45:28,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151bde479d1261469d888c63485f29000e_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631527723/Put/seqid=0 2024-11-15T00:45:28,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741989_1165 (size=31617) 2024-11-15T00:45:28,435 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/B of f762865d4d0d37fcd928550c43e0b373 into 586916ae814e49a595b200fbc72ef27b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:28,436 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:28,436 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/B, priority=13, startTime=1731631528317; duration=0sec 2024-11-15T00:45:28,436 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:28,436 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:B 2024-11-15T00:45:28,436 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:28,446 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/b93d49bee54147c190994497ec689f3e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b93d49bee54147c190994497ec689f3e 2024-11-15T00:45:28,451 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:28,451 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/C is initiating minor compaction (all files) 2024-11-15T00:45:28,451 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/C in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:28,452 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/4bf24f3cea314b60aad4aad7d6e4c88a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/56f9cc5fa524496bb9203fb0c9f96cbd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3adda042f3f942bc95107dba0189d478] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=36.0 K 2024-11-15T00:45:28,456 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/A of f762865d4d0d37fcd928550c43e0b373 into b93d49bee54147c190994497ec689f3e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:28,456 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:28,456 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/A, priority=13, startTime=1731631528316; duration=0sec 2024-11-15T00:45:28,456 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:28,456 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:A 2024-11-15T00:45:28,457 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf24f3cea314b60aad4aad7d6e4c88a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631522592 2024-11-15T00:45:28,459 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 56f9cc5fa524496bb9203fb0c9f96cbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731631525056 2024-11-15T00:45:28,459 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3adda042f3f942bc95107dba0189d478, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731631526401 2024-11-15T00:45:28,473 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:28,474 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/dc218d0517434b51a28bd04ef445b2d7 is 50, key is test_row_0/C:col10/1731631527579/Put/seqid=0 2024-11-15T00:45:28,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741990_1166 (size=12304) 2024-11-15T00:45:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:45:28,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741991_1167 (size=12663) 2024-11-15T00:45:28,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631588560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:28,567 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/dc218d0517434b51a28bd04ef445b2d7 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/dc218d0517434b51a28bd04ef445b2d7 2024-11-15T00:45:28,581 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/C of f762865d4d0d37fcd928550c43e0b373 into dc218d0517434b51a28bd04ef445b2d7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:28,581 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:28,581 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/C, priority=13, startTime=1731631528317; duration=0sec 2024-11-15T00:45:28,581 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:28,581 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:C 2024-11-15T00:45:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631588669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:28,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:28,882 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151bde479d1261469d888c63485f29000e_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151bde479d1261469d888c63485f29000e_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:28,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/db7ec5dacf724918b0b12ad3a0bde778, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:28,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/db7ec5dacf724918b0b12ad3a0bde778 is 175, key is test_row_0/A:col10/1731631527723/Put/seqid=0 2024-11-15T00:45:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631588889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:28,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741992_1168 (size=31105) 2024-11-15T00:45:28,917 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/db7ec5dacf724918b0b12ad3a0bde778 2024-11-15T00:45:28,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/7d60d733ef97410790f0e86129bcf7cb is 50, key is test_row_0/B:col10/1731631527723/Put/seqid=0 2024-11-15T00:45:28,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741993_1169 (size=12151) 2024-11-15T00:45:28,973 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/7d60d733ef97410790f0e86129bcf7cb 2024-11-15T00:45:28,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/c50837345e0d41489d5af68857f7e5d8 is 50, key is test_row_0/C:col10/1731631527723/Put/seqid=0 2024-11-15T00:45:29,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741994_1170 (size=12151) 2024-11-15T00:45:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:45:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631589201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:29,422 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/c50837345e0d41489d5af68857f7e5d8 2024-11-15T00:45:29,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/db7ec5dacf724918b0b12ad3a0bde778 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/db7ec5dacf724918b0b12ad3a0bde778 2024-11-15T00:45:29,447 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/db7ec5dacf724918b0b12ad3a0bde778, entries=150, sequenceid=237, filesize=30.4 K 2024-11-15T00:45:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/7d60d733ef97410790f0e86129bcf7cb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7d60d733ef97410790f0e86129bcf7cb 2024-11-15T00:45:29,466 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7d60d733ef97410790f0e86129bcf7cb, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T00:45:29,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/c50837345e0d41489d5af68857f7e5d8 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/c50837345e0d41489d5af68857f7e5d8 2024-11-15T00:45:29,481 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/c50837345e0d41489d5af68857f7e5d8, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T00:45:29,488 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f762865d4d0d37fcd928550c43e0b373 in 1082ms, sequenceid=237, compaction requested=false 2024-11-15T00:45:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:29,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:29,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T00:45:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T00:45:29,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T00:45:29,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5750 sec 2024-11-15T00:45:29,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.5820 sec 2024-11-15T00:45:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:29,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T00:45:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:29,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:29,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151eed5a89c731439b93d88e21043d97c7_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631529711/Put/seqid=0 2024-11-15T00:45:29,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741995_1171 (size=14794) 2024-11-15T00:45:29,744 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:29,757 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151eed5a89c731439b93d88e21043d97c7_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151eed5a89c731439b93d88e21043d97c7_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:29,758 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/b2eb93751c824618b51e209626957b71, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:29,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/b2eb93751c824618b51e209626957b71 is 175, key is test_row_0/A:col10/1731631529711/Put/seqid=0 2024-11-15T00:45:29,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741996_1172 (size=39749) 2024-11-15T00:45:29,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631589895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:30,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631590009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:30,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:30,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:30,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:45:30,060 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:30,060 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T00:45:30,064 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:45:30,065 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:30,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:45:30,186 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/b2eb93751c824618b51e209626957b71 2024-11-15T00:45:30,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/7dcb6adf5b88461c89a851941c0de36a is 50, key is test_row_0/B:col10/1731631529711/Put/seqid=0 2024-11-15T00:45:30,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741997_1173 (size=12151) 2024-11-15T00:45:30,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:45:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:30,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:30,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631590219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:30,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:30,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,332 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:45:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:45:30,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:45:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:30,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631590532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:30,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:45:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:30,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:30,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:30,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:30,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/7dcb6adf5b88461c89a851941c0de36a 2024-11-15T00:45:30,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/8a20b3f6efac4d58aeb58918c95f49be is 50, key is test_row_0/C:col10/1731631529711/Put/seqid=0 2024-11-15T00:45:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:45:30,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:45:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:30,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:30,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741998_1174 (size=12151) 2024-11-15T00:45:30,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:45:30,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:30,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:30,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:30,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:31,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:45:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:31,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:31,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631591050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:31,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/8a20b3f6efac4d58aeb58918c95f49be 2024-11-15T00:45:31,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:45:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:31,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:45:31,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/b2eb93751c824618b51e209626957b71 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b2eb93751c824618b51e209626957b71 2024-11-15T00:45:31,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b2eb93751c824618b51e209626957b71, entries=200, sequenceid=256, filesize=38.8 K 2024-11-15T00:45:31,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/7dcb6adf5b88461c89a851941c0de36a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7dcb6adf5b88461c89a851941c0de36a 2024-11-15T00:45:31,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7dcb6adf5b88461c89a851941c0de36a, entries=150, sequenceid=256, filesize=11.9 K 2024-11-15T00:45:31,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/8a20b3f6efac4d58aeb58918c95f49be as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8a20b3f6efac4d58aeb58918c95f49be 2024-11-15T00:45:31,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8a20b3f6efac4d58aeb58918c95f49be, entries=150, sequenceid=256, filesize=11.9 K 2024-11-15T00:45:31,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f762865d4d0d37fcd928550c43e0b373 in 1607ms, sequenceid=256, compaction requested=true 2024-11-15T00:45:31,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:31,321 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:31,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:31,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:31,321 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:31,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:31,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:31,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:31,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:31,327 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:31,327 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/A is initiating minor compaction (all files) 2024-11-15T00:45:31,327 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/A in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:31,328 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b93d49bee54147c190994497ec689f3e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/db7ec5dacf724918b0b12ad3a0bde778, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b2eb93751c824618b51e209626957b71] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=100.1 K 2024-11-15T00:45:31,328 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:31,328 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/B is initiating minor compaction (all files) 2024-11-15T00:45:31,328 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:31,328 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/B in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:31,328 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b93d49bee54147c190994497ec689f3e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/db7ec5dacf724918b0b12ad3a0bde778, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b2eb93751c824618b51e209626957b71] 2024-11-15T00:45:31,328 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/586916ae814e49a595b200fbc72ef27b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7d60d733ef97410790f0e86129bcf7cb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7dcb6adf5b88461c89a851941c0de36a] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=36.1 K 2024-11-15T00:45:31,328 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b93d49bee54147c190994497ec689f3e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731631526401 2024-11-15T00:45:31,328 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 586916ae814e49a595b200fbc72ef27b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731631526401 2024-11-15T00:45:31,329 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d60d733ef97410790f0e86129bcf7cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731631527705 2024-11-15T00:45:31,329 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting db7ec5dacf724918b0b12ad3a0bde778, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731631527705 2024-11-15T00:45:31,329 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dcb6adf5b88461c89a851941c0de36a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731631528453 2024-11-15T00:45:31,329 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2eb93751c824618b51e209626957b71, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731631528453 2024-11-15T00:45:31,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:45:31,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:31,345 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T00:45:31,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:31,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:31,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:31,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:31,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:31,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:31,406 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:31,407 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:31,407 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/53ca01c9dd4a441183171e3fd8ac7539 is 50, key is test_row_0/B:col10/1731631529711/Put/seqid=0 2024-11-15T00:45:31,429 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115918ac0ef71cf4c53a90c0985a533a01b_f762865d4d0d37fcd928550c43e0b373 store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:31,431 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115918ac0ef71cf4c53a90c0985a533a01b_f762865d4d0d37fcd928550c43e0b373, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:31,431 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115918ac0ef71cf4c53a90c0985a533a01b_f762865d4d0d37fcd928550c43e0b373 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:31,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111506490d83b40e4c4d93cc71f25ab3fc88_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631529831/Put/seqid=0 2024-11-15T00:45:31,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741999_1175 (size=12765) 2024-11-15T00:45:31,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742001_1177 (size=12454) 2024-11-15T00:45:31,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742000_1176 (size=4469) 2024-11-15T00:45:31,897 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/53ca01c9dd4a441183171e3fd8ac7539 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/53ca01c9dd4a441183171e3fd8ac7539 2024-11-15T00:45:31,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:31,929 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/B of f762865d4d0d37fcd928550c43e0b373 into 53ca01c9dd4a441183171e3fd8ac7539(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:31,929 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:31,929 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/B, priority=13, startTime=1731631531321; duration=0sec 2024-11-15T00:45:31,929 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:31,929 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:B 2024-11-15T00:45:31,929 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:31,941 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:31,941 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/C is initiating minor compaction (all files) 2024-11-15T00:45:31,941 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/C in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:31,942 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/dc218d0517434b51a28bd04ef445b2d7, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/c50837345e0d41489d5af68857f7e5d8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8a20b3f6efac4d58aeb58918c95f49be] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=36.1 K 2024-11-15T00:45:31,945 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#A#compaction#142 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:31,945 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/9cf985519193405895bf6593d8ab44bf is 175, key is test_row_0/A:col10/1731631529711/Put/seqid=0 2024-11-15T00:45:31,947 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111506490d83b40e4c4d93cc71f25ab3fc88_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111506490d83b40e4c4d93cc71f25ab3fc88_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:31,948 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting dc218d0517434b51a28bd04ef445b2d7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731631526401 2024-11-15T00:45:31,960 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c50837345e0d41489d5af68857f7e5d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731631527705 2024-11-15T00:45:31,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/81c975061c984039bde72cad5262d28b, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:31,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/81c975061c984039bde72cad5262d28b is 175, key is test_row_0/A:col10/1731631529831/Put/seqid=0 2024-11-15T00:45:31,969 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a20b3f6efac4d58aeb58918c95f49be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731631528453 2024-11-15T00:45:31,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742002_1178 (size=31719) 2024-11-15T00:45:32,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742003_1179 (size=31255) 2024-11-15T00:45:32,007 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/9cf985519193405895bf6593d8ab44bf as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/9cf985519193405895bf6593d8ab44bf 2024-11-15T00:45:32,007 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/81c975061c984039bde72cad5262d28b 2024-11-15T00:45:32,007 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:32,008 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/e3dc4345f6e0448eadceb53f54a64b67 is 50, key is test_row_0/C:col10/1731631529711/Put/seqid=0 2024-11-15T00:45:32,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/504a77d9f3f04bbc96bfb74b1df21aa2 is 50, key is test_row_0/B:col10/1731631529831/Put/seqid=0 2024-11-15T00:45:32,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742004_1180 (size=12765) 2024-11-15T00:45:32,037 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/A of f762865d4d0d37fcd928550c43e0b373 into 9cf985519193405895bf6593d8ab44bf(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:32,037 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:32,037 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/A, priority=13, startTime=1731631531321; duration=0sec 2024-11-15T00:45:32,038 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:32,038 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:A 2024-11-15T00:45:32,057 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/e3dc4345f6e0448eadceb53f54a64b67 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/e3dc4345f6e0448eadceb53f54a64b67 2024-11-15T00:45:32,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742005_1181 (size=12301) 2024-11-15T00:45:32,075 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/504a77d9f3f04bbc96bfb74b1df21aa2 2024-11-15T00:45:32,078 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/C of f762865d4d0d37fcd928550c43e0b373 into e3dc4345f6e0448eadceb53f54a64b67(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:32,078 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:32,078 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/C, priority=13, startTime=1731631531322; duration=0sec 2024-11-15T00:45:32,078 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:32,078 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:C 2024-11-15T00:45:32,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. as already flushing 2024-11-15T00:45:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:32,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/5919c316a6ac421fb8240016608f5c77 is 50, key is test_row_0/C:col10/1731631529831/Put/seqid=0 2024-11-15T00:45:32,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742006_1182 (size=12301) 2024-11-15T00:45:32,143 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/5919c316a6ac421fb8240016608f5c77 2024-11-15T00:45:32,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/81c975061c984039bde72cad5262d28b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/81c975061c984039bde72cad5262d28b 2024-11-15T00:45:32,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/81c975061c984039bde72cad5262d28b, entries=150, sequenceid=272, filesize=30.5 K 2024-11-15T00:45:32,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/504a77d9f3f04bbc96bfb74b1df21aa2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/504a77d9f3f04bbc96bfb74b1df21aa2 2024-11-15T00:45:32,197 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/504a77d9f3f04bbc96bfb74b1df21aa2, entries=150, sequenceid=272, filesize=12.0 K 2024-11-15T00:45:32,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/5919c316a6ac421fb8240016608f5c77 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/5919c316a6ac421fb8240016608f5c77 2024-11-15T00:45:32,211 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/5919c316a6ac421fb8240016608f5c77, entries=150, sequenceid=272, filesize=12.0 K 2024-11-15T00:45:32,212 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=53.67 KB/54960 for f762865d4d0d37fcd928550c43e0b373 in 866ms, sequenceid=272, compaction requested=false 2024-11-15T00:45:32,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:32,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:32,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T00:45:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T00:45:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:45:32,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T00:45:32,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1520 sec 2024-11-15T00:45:32,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.1660 sec 2024-11-15T00:45:32,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:45:32,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:32,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:32,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:32,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:32,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:32,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:32,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150a196d33aa8a4b169fcd9aa29e433c37_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631532230/Put/seqid=0 2024-11-15T00:45:32,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742007_1183 (size=14994) 2024-11-15T00:45:32,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631592469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:32,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:32,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631592579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:32,694 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:32,702 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150a196d33aa8a4b169fcd9aa29e433c37_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150a196d33aa8a4b169fcd9aa29e433c37_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:32,705 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/2841ee2f9fe8446aaadcb19b179cb8fd, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:32,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/2841ee2f9fe8446aaadcb19b179cb8fd is 175, key is test_row_0/A:col10/1731631532230/Put/seqid=0 2024-11-15T00:45:32,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742008_1184 (size=39949) 2024-11-15T00:45:32,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631592789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:33,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631593100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:33,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:33,137 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/2841ee2f9fe8446aaadcb19b179cb8fd 2024-11-15T00:45:33,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/8dff6523138a4f99a254b1d0cfbd94eb is 50, key is test_row_0/B:col10/1731631532230/Put/seqid=0 2024-11-15T00:45:33,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742009_1185 (size=12301) 2024-11-15T00:45:33,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/8dff6523138a4f99a254b1d0cfbd94eb 2024-11-15T00:45:33,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/b95514fe2f5a4d2f9f1c0ceebe05c499 is 50, key is test_row_0/C:col10/1731631532230/Put/seqid=0 2024-11-15T00:45:33,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742010_1186 (size=12301) 2024-11-15T00:45:33,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57528 deadline: 1731631593609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:33,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/b95514fe2f5a4d2f9f1c0ceebe05c499 2024-11-15T00:45:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/2841ee2f9fe8446aaadcb19b179cb8fd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/2841ee2f9fe8446aaadcb19b179cb8fd 2024-11-15T00:45:33,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/2841ee2f9fe8446aaadcb19b179cb8fd, entries=200, sequenceid=288, filesize=39.0 K 2024-11-15T00:45:33,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/8dff6523138a4f99a254b1d0cfbd94eb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/8dff6523138a4f99a254b1d0cfbd94eb 2024-11-15T00:45:33,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/8dff6523138a4f99a254b1d0cfbd94eb, entries=150, sequenceid=288, filesize=12.0 K 2024-11-15T00:45:33,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/b95514fe2f5a4d2f9f1c0ceebe05c499 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/b95514fe2f5a4d2f9f1c0ceebe05c499 2024-11-15T00:45:33,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/b95514fe2f5a4d2f9f1c0ceebe05c499, entries=150, sequenceid=288, filesize=12.0 K 2024-11-15T00:45:33,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f762865d4d0d37fcd928550c43e0b373 in 1447ms, sequenceid=288, compaction requested=true 2024-11-15T00:45:33,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:33,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:33,687 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:33,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:33,687 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:33,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:33,688 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:33,689 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/A is initiating minor compaction (all files) 2024-11-15T00:45:33,689 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:33,689 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/A in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:33,689 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/B is initiating minor compaction (all files) 2024-11-15T00:45:33,689 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/B in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:33,689 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/53ca01c9dd4a441183171e3fd8ac7539, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/504a77d9f3f04bbc96bfb74b1df21aa2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/8dff6523138a4f99a254b1d0cfbd94eb] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=36.5 K 2024-11-15T00:45:33,689 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/9cf985519193405895bf6593d8ab44bf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/81c975061c984039bde72cad5262d28b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/2841ee2f9fe8446aaadcb19b179cb8fd] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=100.5 K 2024-11-15T00:45:33,689 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:33,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:33,689 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/9cf985519193405895bf6593d8ab44bf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/81c975061c984039bde72cad5262d28b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/2841ee2f9fe8446aaadcb19b179cb8fd] 2024-11-15T00:45:33,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:33,690 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cf985519193405895bf6593d8ab44bf, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731631528453 2024-11-15T00:45:33,690 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 53ca01c9dd4a441183171e3fd8ac7539, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731631528453 2024-11-15T00:45:33,690 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81c975061c984039bde72cad5262d28b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731631529791 2024-11-15T00:45:33,691 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 504a77d9f3f04bbc96bfb74b1df21aa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731631529791 2024-11-15T00:45:33,691 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2841ee2f9fe8446aaadcb19b179cb8fd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731631532125 2024-11-15T00:45:33,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:33,692 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dff6523138a4f99a254b1d0cfbd94eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731631532176 2024-11-15T00:45:33,713 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:33,725 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:33,726 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/fc7ec6345fd743b4be0034d9af118381 is 50, key is test_row_0/B:col10/1731631532230/Put/seqid=0 2024-11-15T00:45:33,738 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115a3dc363a40ca45e59bc208936e254283_f762865d4d0d37fcd928550c43e0b373 store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:33,741 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115a3dc363a40ca45e59bc208936e254283_f762865d4d0d37fcd928550c43e0b373, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:33,741 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a3dc363a40ca45e59bc208936e254283_f762865d4d0d37fcd928550c43e0b373 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:33,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742011_1187 (size=13017) 2024-11-15T00:45:33,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742012_1188 (size=4469) 2024-11-15T00:45:33,827 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#A#compaction#150 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:33,827 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/f98a9df5245c49e39ddcdc7611a9442c is 175, key is test_row_0/A:col10/1731631532230/Put/seqid=0 2024-11-15T00:45:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:33,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:45:33,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:33,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:33,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:33,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:33,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:33,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:33,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742013_1189 (size=31971) 2024-11-15T00:45:33,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158a2c19b2e7854ba9a47ff2fb7c774012_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631533846/Put/seqid=0 2024-11-15T00:45:33,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631593900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:33,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742014_1190 (size=14994) 2024-11-15T00:45:33,916 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:33,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631593909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:33,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631593911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:33,954 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158a2c19b2e7854ba9a47ff2fb7c774012_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158a2c19b2e7854ba9a47ff2fb7c774012_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:33,958 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/37e46c9ae80e41f5b83102053f1a2a29, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:33,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/37e46c9ae80e41f5b83102053f1a2a29 is 175, key is test_row_0/A:col10/1731631533846/Put/seqid=0 2024-11-15T00:45:33,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57538 deadline: 1731631593951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:33,960 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in f762865d4d0d37fcd928550c43e0b373 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18403 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:33,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742015_1191 (size=39949) 2024-11-15T00:45:33,988 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/37e46c9ae80e41f5b83102053f1a2a29 2024-11-15T00:45:34,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/75ad05fe93e94dd4853a5e3eef9435c2 is 50, key is test_row_0/B:col10/1731631533846/Put/seqid=0 2024-11-15T00:45:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57554 deadline: 1731631594020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:34,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742016_1192 (size=12301) 2024-11-15T00:45:34,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/75ad05fe93e94dd4853a5e3eef9435c2 2024-11-15T00:45:34,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/a285e42d75fe4ec781282e5d6beac63f is 50, key is test_row_0/C:col10/1731631533846/Put/seqid=0 2024-11-15T00:45:34,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57566 deadline: 1731631594059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:34,067 INFO [Thread-551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-551. 2024-11-15T00:45:34,067 DEBUG [Thread-551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:34,067 DEBUG [Thread-551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:34,067 DEBUG [Thread-551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,067 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:34,068 INFO [Thread-557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-557. 2024-11-15T00:45:34,068 DEBUG [Thread-557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:34,068 DEBUG [Thread-557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,068 DEBUG [Thread-557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:34,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57568 deadline: 1731631594066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:45:34,075 INFO [Thread-553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-553. 2024-11-15T00:45:34,075 DEBUG [Thread-553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:34,075 DEBUG [Thread-553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,076 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:34,076 DEBUG [Thread-553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,079 INFO [Thread-555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-555. 2024-11-15T00:45:34,079 DEBUG [Thread-555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:34,079 DEBUG [Thread-555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,080 DEBUG [Thread-555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:34,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742017_1193 (size=12301) 2024-11-15T00:45:34,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/a285e42d75fe4ec781282e5d6beac63f 2024-11-15T00:45:34,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/37e46c9ae80e41f5b83102053f1a2a29 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/37e46c9ae80e41f5b83102053f1a2a29 2024-11-15T00:45:34,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/37e46c9ae80e41f5b83102053f1a2a29, entries=200, sequenceid=308, filesize=39.0 K 2024-11-15T00:45:34,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/75ad05fe93e94dd4853a5e3eef9435c2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/75ad05fe93e94dd4853a5e3eef9435c2 2024-11-15T00:45:34,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/75ad05fe93e94dd4853a5e3eef9435c2, entries=150, sequenceid=308, filesize=12.0 K 2024-11-15T00:45:34,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/a285e42d75fe4ec781282e5d6beac63f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/a285e42d75fe4ec781282e5d6beac63f 2024-11-15T00:45:34,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/a285e42d75fe4ec781282e5d6beac63f, entries=150, sequenceid=308, filesize=12.0 K 2024-11-15T00:45:34,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f762865d4d0d37fcd928550c43e0b373 in 264ms, sequenceid=308, compaction requested=true 2024-11-15T00:45:34,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:34,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:A, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:34,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:45:34,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:B, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:34,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T00:45:34,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:34,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T00:45:34,219 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/fc7ec6345fd743b4be0034d9af118381 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/fc7ec6345fd743b4be0034d9af118381 2024-11-15T00:45:34,224 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/B of f762865d4d0d37fcd928550c43e0b373 into fc7ec6345fd743b4be0034d9af118381(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:34,225 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:34,225 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/B, priority=13, startTime=1731631533687; duration=0sec 2024-11-15T00:45:34,225 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T00:45:34,225 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:B 2024-11-15T00:45:34,225 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-15T00:45:34,227 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:45:34,227 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:45:34,228 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. because compaction request was cancelled 2024-11-15T00:45:34,228 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:A 2024-11-15T00:45:34,228 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:45:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:34,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T00:45:34,231 INFO [Thread-540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-540. 2024-11-15T00:45:34,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:34,231 DEBUG [Thread-540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:34,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:34,231 DEBUG [Thread-540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:34,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:34,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:34,231 DEBUG [Thread-540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:34,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:34,232 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:45:34,232 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/C is initiating minor compaction (all files) 2024-11-15T00:45:34,232 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/C in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:34,232 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/e3dc4345f6e0448eadceb53f54a64b67, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/5919c316a6ac421fb8240016608f5c77, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/b95514fe2f5a4d2f9f1c0ceebe05c499, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/a285e42d75fe4ec781282e5d6beac63f] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=48.5 K 2024-11-15T00:45:34,233 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e3dc4345f6e0448eadceb53f54a64b67, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731631528453 2024-11-15T00:45:34,234 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 5919c316a6ac421fb8240016608f5c77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731631529791 2024-11-15T00:45:34,234 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting b95514fe2f5a4d2f9f1c0ceebe05c499, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731631532176 2024-11-15T00:45:34,234 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a285e42d75fe4ec781282e5d6beac63f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731631532429 2024-11-15T00:45:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:45:34,240 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:34,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f8b8fe69b3248649548954f130c3796_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631534229/Put/seqid=0 2024-11-15T00:45:34,260 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#C#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:34,261 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/9875822bbe3e42ff811e7683533cc45e is 50, key is test_row_0/C:col10/1731631533846/Put/seqid=0 2024-11-15T00:45:34,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742018_1194 (size=12454) 2024-11-15T00:45:34,265 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:34,271 INFO [Thread-546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-546. 2024-11-15T00:45:34,271 DEBUG [Thread-546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:34,271 DEBUG [Thread-546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,271 DEBUG [Thread-546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,275 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/f98a9df5245c49e39ddcdc7611a9442c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/f98a9df5245c49e39ddcdc7611a9442c 2024-11-15T00:45:34,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:34,282 INFO [Thread-544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-544. 2024-11-15T00:45:34,282 DEBUG [Thread-544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:34,282 DEBUG [Thread-544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,282 DEBUG [Thread-544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:34,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f8b8fe69b3248649548954f130c3796_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f8b8fe69b3248649548954f130c3796_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:34,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742019_1195 (size=13051) 2024-11-15T00:45:34,286 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/651f6a69399d43bc99d307d32e1dfa89, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:34,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/651f6a69399d43bc99d307d32e1dfa89 is 175, key is test_row_0/A:col10/1731631534229/Put/seqid=0 2024-11-15T00:45:34,289 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/A of f762865d4d0d37fcd928550c43e0b373 into f98a9df5245c49e39ddcdc7611a9442c(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:34,289 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:34,289 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/A, priority=13, startTime=1731631533687; duration=0sec 2024-11-15T00:45:34,289 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:45:34,290 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:A 2024-11-15T00:45:34,290 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:B 2024-11-15T00:45:34,290 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-15T00:45:34,290 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:45:34,290 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:45:34,290 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. because compaction request was cancelled 2024-11-15T00:45:34,290 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:C 2024-11-15T00:45:34,290 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T00:45:34,292 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:45:34,292 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:45:34,292 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. because compaction request was cancelled 2024-11-15T00:45:34,292 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:B 2024-11-15T00:45:34,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742020_1196 (size=31255) 2024-11-15T00:45:34,306 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/651f6a69399d43bc99d307d32e1dfa89 2024-11-15T00:45:34,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/b0836ea2a0234f45bdbd58ac380e36d7 is 50, key is test_row_0/B:col10/1731631534229/Put/seqid=0 2024-11-15T00:45:34,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742021_1197 (size=12301) 2024-11-15T00:45:34,631 INFO [Thread-542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-542. 2024-11-15T00:45:34,631 DEBUG [Thread-542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:34,631 DEBUG [Thread-542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,631 DEBUG [Thread-542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:34,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:34,693 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/9875822bbe3e42ff811e7683533cc45e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/9875822bbe3e42ff811e7683533cc45e 2024-11-15T00:45:34,701 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/C of f762865d4d0d37fcd928550c43e0b373 into 9875822bbe3e42ff811e7683533cc45e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:34,701 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:34,701 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/C, priority=12, startTime=1731631534114; duration=0sec 2024-11-15T00:45:34,701 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:34,701 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:C 2024-11-15T00:45:34,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/b0836ea2a0234f45bdbd58ac380e36d7 2024-11-15T00:45:34,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/cb2aa0a63f9c4f2384bea7b348fff023 is 50, key is test_row_0/C:col10/1731631534229/Put/seqid=0 2024-11-15T00:45:34,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742022_1198 (size=12301) 2024-11-15T00:45:35,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/cb2aa0a63f9c4f2384bea7b348fff023 2024-11-15T00:45:35,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/651f6a69399d43bc99d307d32e1dfa89 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/651f6a69399d43bc99d307d32e1dfa89 2024-11-15T00:45:35,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/651f6a69399d43bc99d307d32e1dfa89, entries=150, sequenceid=323, filesize=30.5 K 2024-11-15T00:45:35,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/b0836ea2a0234f45bdbd58ac380e36d7 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/b0836ea2a0234f45bdbd58ac380e36d7 2024-11-15T00:45:35,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/b0836ea2a0234f45bdbd58ac380e36d7, entries=150, sequenceid=323, filesize=12.0 K 2024-11-15T00:45:35,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/cb2aa0a63f9c4f2384bea7b348fff023 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/cb2aa0a63f9c4f2384bea7b348fff023 2024-11-15T00:45:35,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/cb2aa0a63f9c4f2384bea7b348fff023, entries=150, sequenceid=323, filesize=12.0 K 2024-11-15T00:45:35,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for f762865d4d0d37fcd928550c43e0b373 in 958ms, sequenceid=323, compaction requested=true 2024-11-15T00:45:35,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:35,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:35,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:35,189 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:35,189 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:35,190 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:35,190 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:35,190 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/A is initiating minor compaction (all files) 2024-11-15T00:45:35,190 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): f762865d4d0d37fcd928550c43e0b373/B is initiating minor compaction (all files) 2024-11-15T00:45:35,191 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/A in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:35,191 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f762865d4d0d37fcd928550c43e0b373/B in TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:35,191 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/f98a9df5245c49e39ddcdc7611a9442c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/37e46c9ae80e41f5b83102053f1a2a29, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/651f6a69399d43bc99d307d32e1dfa89] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=100.8 K 2024-11-15T00:45:35,191 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/fc7ec6345fd743b4be0034d9af118381, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/75ad05fe93e94dd4853a5e3eef9435c2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/b0836ea2a0234f45bdbd58ac380e36d7] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp, totalSize=36.7 K 2024-11-15T00:45:35,191 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:35,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:35,191 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/f98a9df5245c49e39ddcdc7611a9442c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/37e46c9ae80e41f5b83102053f1a2a29, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/651f6a69399d43bc99d307d32e1dfa89] 2024-11-15T00:45:35,191 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7ec6345fd743b4be0034d9af118381, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731631532176 2024-11-15T00:45:35,191 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting f98a9df5245c49e39ddcdc7611a9442c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731631532176 2024-11-15T00:45:35,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:35,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f762865d4d0d37fcd928550c43e0b373:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:35,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:35,192 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 75ad05fe93e94dd4853a5e3eef9435c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731631532429 2024-11-15T00:45:35,192 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e46c9ae80e41f5b83102053f1a2a29, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731631532429 2024-11-15T00:45:35,193 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting b0836ea2a0234f45bdbd58ac380e36d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731631533868 2024-11-15T00:45:35,193 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 651f6a69399d43bc99d307d32e1dfa89, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731631533868 2024-11-15T00:45:35,242 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#B#compaction#159 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:35,243 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/a14a01e5265f4441a26b77f3dad61daf is 50, key is test_row_0/B:col10/1731631534229/Put/seqid=0 2024-11-15T00:45:35,257 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:35,289 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111543afe0da6b154e82896db99eef85b57a_f762865d4d0d37fcd928550c43e0b373 store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:35,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742023_1199 (size=13119) 2024-11-15T00:45:35,304 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/a14a01e5265f4441a26b77f3dad61daf as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/a14a01e5265f4441a26b77f3dad61daf 2024-11-15T00:45:35,315 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/B of f762865d4d0d37fcd928550c43e0b373 into a14a01e5265f4441a26b77f3dad61daf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:35,315 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:35,315 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/B, priority=13, startTime=1731631535189; duration=0sec 2024-11-15T00:45:35,316 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:35,316 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:B 2024-11-15T00:45:35,316 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T00:45:35,317 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:45:35,317 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:45:35,317 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. because compaction request was cancelled 2024-11-15T00:45:35,317 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:C 2024-11-15T00:45:35,342 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111543afe0da6b154e82896db99eef85b57a_f762865d4d0d37fcd928550c43e0b373, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:35,342 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111543afe0da6b154e82896db99eef85b57a_f762865d4d0d37fcd928550c43e0b373 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:35,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742024_1200 (size=4469) 2024-11-15T00:45:35,784 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f762865d4d0d37fcd928550c43e0b373#A#compaction#160 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:35,785 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/684e1908ecc3478893cb135dc7bd3283 is 175, key is test_row_0/A:col10/1731631534229/Put/seqid=0 2024-11-15T00:45:35,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742025_1201 (size=32073) 2024-11-15T00:45:36,196 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/684e1908ecc3478893cb135dc7bd3283 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/684e1908ecc3478893cb135dc7bd3283 2024-11-15T00:45:36,203 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f762865d4d0d37fcd928550c43e0b373/A of f762865d4d0d37fcd928550c43e0b373 into 684e1908ecc3478893cb135dc7bd3283(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-15T00:45:36,203 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f762865d4d0d37fcd928550c43e0b373: 2024-11-15T00:45:36,203 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., storeName=f762865d4d0d37fcd928550c43e0b373/A, priority=13, startTime=1731631535189; duration=1sec 2024-11-15T00:45:36,204 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:36,204 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f762865d4d0d37fcd928550c43e0b373:A 2024-11-15T00:45:44,003 INFO [Thread-548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-548. 2024-11-15T00:45:44,003 DEBUG [Thread-548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:45:44,003 DEBUG [Thread-548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:44,003 DEBUG [Thread-548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:44,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:44,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:45:44,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-15T00:45:44,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 177 2024-11-15T00:45:44,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-15T00:45:44,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-15T00:45:44,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-15T00:45:44,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:45:44,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1725 2024-11-15T00:45:44,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1775 2024-11-15T00:45:44,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:45:44,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 796 2024-11-15T00:45:44,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2383 rows 2024-11-15T00:45:44,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 796 2024-11-15T00:45:44,004 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2388 rows 2024-11-15T00:45:44,004 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:45:44,004 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:45:44,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:44,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:44,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:45:44,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:45:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:45:44,009 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631544009"}]},"ts":"1731631544009"} 2024-11-15T00:45:44,011 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:45:44,011 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:45:44,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:45:44,015 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, UNASSIGN}] 2024-11-15T00:45:44,016 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, UNASSIGN 2024-11-15T00:45:44,017 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=f762865d4d0d37fcd928550c43e0b373, regionState=CLOSING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:44,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, UNASSIGN because future has completed 2024-11-15T00:45:44,019 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:45:44,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:45:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:45:44,172 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:44,172 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:45:44,172 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing f762865d4d0d37fcd928550c43e0b373, disabling compactions & flushes 2024-11-15T00:45:44,172 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:44,172 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:44,172 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. after waiting 0 ms 2024-11-15T00:45:44,172 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:44,172 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing f762865d4d0d37fcd928550c43e0b373 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-15T00:45:44,173 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=A 2024-11-15T00:45:44,173 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:44,173 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=B 2024-11-15T00:45:44,173 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:44,173 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f762865d4d0d37fcd928550c43e0b373, store=C 2024-11-15T00:45:44,173 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:44,184 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bc197e2baf9945aca4e6aa8822ad5b76_f762865d4d0d37fcd928550c43e0b373 is 50, key is test_row_0/A:col10/1731631534280/Put/seqid=0 2024-11-15T00:45:44,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742026_1202 (size=12454) 2024-11-15T00:45:44,210 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:44,227 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bc197e2baf9945aca4e6aa8822ad5b76_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bc197e2baf9945aca4e6aa8822ad5b76_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:44,228 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/5f3600772a7c4d0da13e8bc1335781f0, store: [table=TestAcidGuarantees family=A region=f762865d4d0d37fcd928550c43e0b373] 2024-11-15T00:45:44,228 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/5f3600772a7c4d0da13e8bc1335781f0 is 175, key is test_row_0/A:col10/1731631534280/Put/seqid=0 2024-11-15T00:45:44,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742027_1203 (size=31255) 2024-11-15T00:45:44,243 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/5f3600772a7c4d0da13e8bc1335781f0 2024-11-15T00:45:44,256 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2c8733e39fb747b6b1b56e6ce579d713 is 50, key is test_row_0/B:col10/1731631534280/Put/seqid=0 2024-11-15T00:45:44,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742028_1204 (size=12301) 2024-11-15T00:45:44,272 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2c8733e39fb747b6b1b56e6ce579d713 2024-11-15T00:45:44,293 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/dbdfb23a52e141d28f8c32dc7c303378 is 50, key is test_row_0/C:col10/1731631534280/Put/seqid=0 2024-11-15T00:45:44,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742029_1205 (size=12301) 2024-11-15T00:45:44,312 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/dbdfb23a52e141d28f8c32dc7c303378 2024-11-15T00:45:44,328 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/A/5f3600772a7c4d0da13e8bc1335781f0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/5f3600772a7c4d0da13e8bc1335781f0 2024-11-15T00:45:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:45:44,344 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/5f3600772a7c4d0da13e8bc1335781f0, entries=150, sequenceid=334, filesize=30.5 K 2024-11-15T00:45:44,345 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/B/2c8733e39fb747b6b1b56e6ce579d713 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2c8733e39fb747b6b1b56e6ce579d713 2024-11-15T00:45:44,352 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2c8733e39fb747b6b1b56e6ce579d713, entries=150, sequenceid=334, filesize=12.0 K 2024-11-15T00:45:44,354 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/.tmp/C/dbdfb23a52e141d28f8c32dc7c303378 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/dbdfb23a52e141d28f8c32dc7c303378 2024-11-15T00:45:44,369 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/dbdfb23a52e141d28f8c32dc7c303378, entries=150, sequenceid=334, filesize=12.0 K 2024-11-15T00:45:44,380 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f762865d4d0d37fcd928550c43e0b373 in 208ms, sequenceid=334, compaction requested=true 2024-11-15T00:45:44,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/90544f73eb2e4b0aa3300c8436a958ec, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/7fe24db973df4c8a82cc4233ca6e52bb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/a83af7d89a154178bfdf255b841dc8dd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e8065565998248929934e71d628d5a5b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/6f70eed574b0433a8b614221be92d319, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/0b194f4f12874314ba80e44f4fee8dbc, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e9f66da13f4e466f9174db94a5ad0b23, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/24e614015e914e25ad8bd18301043531, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/ffaa0d74a40848368c6cc324f600caf9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/be50547b154041ceba96a514f3d1f96c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/255adccb999641eb9074e6cc342cb25a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/4917526c3148468e89214bd1610e4cd8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/c4bcbe2bf3b9496789fe7022b38da9d1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/16628b82ac154e3ea0bf0025a697b67a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/202cfb2c1e824894a873f63ee463218d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b93d49bee54147c190994497ec689f3e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/db7ec5dacf724918b0b12ad3a0bde778, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b2eb93751c824618b51e209626957b71, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/9cf985519193405895bf6593d8ab44bf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/81c975061c984039bde72cad5262d28b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/2841ee2f9fe8446aaadcb19b179cb8fd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/f98a9df5245c49e39ddcdc7611a9442c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/37e46c9ae80e41f5b83102053f1a2a29, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/651f6a69399d43bc99d307d32e1dfa89] to archive 2024-11-15T00:45:44,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:45:44,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/90544f73eb2e4b0aa3300c8436a958ec to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/90544f73eb2e4b0aa3300c8436a958ec 2024-11-15T00:45:44,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/7fe24db973df4c8a82cc4233ca6e52bb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/7fe24db973df4c8a82cc4233ca6e52bb 2024-11-15T00:45:44,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/a83af7d89a154178bfdf255b841dc8dd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/a83af7d89a154178bfdf255b841dc8dd 2024-11-15T00:45:44,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e8065565998248929934e71d628d5a5b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e8065565998248929934e71d628d5a5b 2024-11-15T00:45:44,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/6f70eed574b0433a8b614221be92d319 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/6f70eed574b0433a8b614221be92d319 2024-11-15T00:45:44,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/0b194f4f12874314ba80e44f4fee8dbc to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/0b194f4f12874314ba80e44f4fee8dbc 2024-11-15T00:45:44,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e9f66da13f4e466f9174db94a5ad0b23 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/e9f66da13f4e466f9174db94a5ad0b23 2024-11-15T00:45:44,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/24e614015e914e25ad8bd18301043531 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/24e614015e914e25ad8bd18301043531 2024-11-15T00:45:44,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/ffaa0d74a40848368c6cc324f600caf9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/ffaa0d74a40848368c6cc324f600caf9 2024-11-15T00:45:44,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/be50547b154041ceba96a514f3d1f96c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/be50547b154041ceba96a514f3d1f96c 2024-11-15T00:45:44,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/255adccb999641eb9074e6cc342cb25a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/255adccb999641eb9074e6cc342cb25a 2024-11-15T00:45:44,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/4917526c3148468e89214bd1610e4cd8 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/4917526c3148468e89214bd1610e4cd8 2024-11-15T00:45:44,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/c4bcbe2bf3b9496789fe7022b38da9d1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/c4bcbe2bf3b9496789fe7022b38da9d1 2024-11-15T00:45:44,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/16628b82ac154e3ea0bf0025a697b67a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/16628b82ac154e3ea0bf0025a697b67a 2024-11-15T00:45:44,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/202cfb2c1e824894a873f63ee463218d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/202cfb2c1e824894a873f63ee463218d 2024-11-15T00:45:44,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b93d49bee54147c190994497ec689f3e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b93d49bee54147c190994497ec689f3e 2024-11-15T00:45:44,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/db7ec5dacf724918b0b12ad3a0bde778 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/db7ec5dacf724918b0b12ad3a0bde778 2024-11-15T00:45:44,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b2eb93751c824618b51e209626957b71 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/b2eb93751c824618b51e209626957b71 2024-11-15T00:45:44,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/9cf985519193405895bf6593d8ab44bf to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/9cf985519193405895bf6593d8ab44bf 2024-11-15T00:45:44,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/81c975061c984039bde72cad5262d28b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/81c975061c984039bde72cad5262d28b 2024-11-15T00:45:44,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/2841ee2f9fe8446aaadcb19b179cb8fd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/2841ee2f9fe8446aaadcb19b179cb8fd 2024-11-15T00:45:44,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/f98a9df5245c49e39ddcdc7611a9442c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/f98a9df5245c49e39ddcdc7611a9442c 2024-11-15T00:45:44,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/37e46c9ae80e41f5b83102053f1a2a29 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/37e46c9ae80e41f5b83102053f1a2a29 2024-11-15T00:45:44,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/651f6a69399d43bc99d307d32e1dfa89 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/651f6a69399d43bc99d307d32e1dfa89 2024-11-15T00:45:44,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2d5eaa105f064bfeabba99c59515d573, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/10519e3d8bce4b279fa6ca2f1a08a64a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ebaac84af0bb498e8b9987c33c8f98b7, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2776a135c6b0473aaf31711e905f819b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5e18d93e07494d60af708f29b22aca84, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/d42b76d3d66042bc8988a74ff91a41db, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2bfcab08730d4fcb919a84acb2a58233, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/444c60080c1b499291ad9d28078631b9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/bd83c877ec1c433da174670446302512, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/610f0f3bf18e426d93c314ff4645d561, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ddf754736d844768b1e6032afd0d72d9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/c1a6b480a3ea446285083558ee4db781, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/986a442dc00242d9bc52b7581ee73aa2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/f2ffcde84467462880d522c0c764a2df, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/586916ae814e49a595b200fbc72ef27b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5cf10224bba6439a9495f3686c57bd36, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7d60d733ef97410790f0e86129bcf7cb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/53ca01c9dd4a441183171e3fd8ac7539, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7dcb6adf5b88461c89a851941c0de36a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/504a77d9f3f04bbc96bfb74b1df21aa2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/fc7ec6345fd743b4be0034d9af118381, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/8dff6523138a4f99a254b1d0cfbd94eb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/75ad05fe93e94dd4853a5e3eef9435c2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/b0836ea2a0234f45bdbd58ac380e36d7] to archive 2024-11-15T00:45:44,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:45:44,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2d5eaa105f064bfeabba99c59515d573 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2d5eaa105f064bfeabba99c59515d573 2024-11-15T00:45:44,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/10519e3d8bce4b279fa6ca2f1a08a64a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/10519e3d8bce4b279fa6ca2f1a08a64a 2024-11-15T00:45:44,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ebaac84af0bb498e8b9987c33c8f98b7 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ebaac84af0bb498e8b9987c33c8f98b7 2024-11-15T00:45:44,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2776a135c6b0473aaf31711e905f819b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2776a135c6b0473aaf31711e905f819b 2024-11-15T00:45:44,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5e18d93e07494d60af708f29b22aca84 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5e18d93e07494d60af708f29b22aca84 2024-11-15T00:45:44,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/d42b76d3d66042bc8988a74ff91a41db to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/d42b76d3d66042bc8988a74ff91a41db 2024-11-15T00:45:44,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2bfcab08730d4fcb919a84acb2a58233 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2bfcab08730d4fcb919a84acb2a58233 2024-11-15T00:45:44,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/444c60080c1b499291ad9d28078631b9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/444c60080c1b499291ad9d28078631b9 2024-11-15T00:45:44,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/bd83c877ec1c433da174670446302512 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/bd83c877ec1c433da174670446302512 2024-11-15T00:45:44,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/610f0f3bf18e426d93c314ff4645d561 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/610f0f3bf18e426d93c314ff4645d561 2024-11-15T00:45:44,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ddf754736d844768b1e6032afd0d72d9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/ddf754736d844768b1e6032afd0d72d9 2024-11-15T00:45:44,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/c1a6b480a3ea446285083558ee4db781 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/c1a6b480a3ea446285083558ee4db781 2024-11-15T00:45:44,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/986a442dc00242d9bc52b7581ee73aa2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/986a442dc00242d9bc52b7581ee73aa2 2024-11-15T00:45:44,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/f2ffcde84467462880d522c0c764a2df to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/f2ffcde84467462880d522c0c764a2df 2024-11-15T00:45:44,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/586916ae814e49a595b200fbc72ef27b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/586916ae814e49a595b200fbc72ef27b 2024-11-15T00:45:44,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5cf10224bba6439a9495f3686c57bd36 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/5cf10224bba6439a9495f3686c57bd36 2024-11-15T00:45:44,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7d60d733ef97410790f0e86129bcf7cb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7d60d733ef97410790f0e86129bcf7cb 2024-11-15T00:45:44,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/53ca01c9dd4a441183171e3fd8ac7539 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/53ca01c9dd4a441183171e3fd8ac7539 2024-11-15T00:45:44,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7dcb6adf5b88461c89a851941c0de36a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/7dcb6adf5b88461c89a851941c0de36a 2024-11-15T00:45:44,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/504a77d9f3f04bbc96bfb74b1df21aa2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/504a77d9f3f04bbc96bfb74b1df21aa2 2024-11-15T00:45:44,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/fc7ec6345fd743b4be0034d9af118381 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/fc7ec6345fd743b4be0034d9af118381 2024-11-15T00:45:44,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/8dff6523138a4f99a254b1d0cfbd94eb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/8dff6523138a4f99a254b1d0cfbd94eb 2024-11-15T00:45:44,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/75ad05fe93e94dd4853a5e3eef9435c2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/75ad05fe93e94dd4853a5e3eef9435c2 2024-11-15T00:45:44,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/b0836ea2a0234f45bdbd58ac380e36d7 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/b0836ea2a0234f45bdbd58ac380e36d7 2024-11-15T00:45:44,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/6b41657ed28f415fa13a8d86538f2aac, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/de29f95e193842e295c6e7bae36d298b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/e0b389200e7d4f9da320ae3c8c877dde, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/14d1cd70f88c4ae79aaa1933b5936bf5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/603afdb7733a4beaae60add27c09da8c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/d758e77250274bb9811e011c15fe5ec7, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/9011e09bd9c348319f50a5417a8d7d08, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/676dd97850714a2db611606e4bae84b0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3d48142371674e1d9b78de4577dce511, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/aafa51c8a9ae41ec8c20cc93f7885a2d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8332d3a26e11420eaba47f005c170db6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/4bf24f3cea314b60aad4aad7d6e4c88a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/52ef6f7f4685488cbcfaecd6bbb67cfd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/56f9cc5fa524496bb9203fb0c9f96cbd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/dc218d0517434b51a28bd04ef445b2d7, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3adda042f3f942bc95107dba0189d478, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/c50837345e0d41489d5af68857f7e5d8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/e3dc4345f6e0448eadceb53f54a64b67, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8a20b3f6efac4d58aeb58918c95f49be, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/5919c316a6ac421fb8240016608f5c77, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/b95514fe2f5a4d2f9f1c0ceebe05c499, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/a285e42d75fe4ec781282e5d6beac63f] to archive 2024-11-15T00:45:44,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:45:44,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/6b41657ed28f415fa13a8d86538f2aac to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/6b41657ed28f415fa13a8d86538f2aac 2024-11-15T00:45:44,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/de29f95e193842e295c6e7bae36d298b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/de29f95e193842e295c6e7bae36d298b 2024-11-15T00:45:44,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/e0b389200e7d4f9da320ae3c8c877dde to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/e0b389200e7d4f9da320ae3c8c877dde 2024-11-15T00:45:44,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/14d1cd70f88c4ae79aaa1933b5936bf5 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/14d1cd70f88c4ae79aaa1933b5936bf5 2024-11-15T00:45:44,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/603afdb7733a4beaae60add27c09da8c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/603afdb7733a4beaae60add27c09da8c 2024-11-15T00:45:44,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/d758e77250274bb9811e011c15fe5ec7 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/d758e77250274bb9811e011c15fe5ec7 2024-11-15T00:45:44,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/9011e09bd9c348319f50a5417a8d7d08 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/9011e09bd9c348319f50a5417a8d7d08 2024-11-15T00:45:44,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/676dd97850714a2db611606e4bae84b0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/676dd97850714a2db611606e4bae84b0 2024-11-15T00:45:44,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3d48142371674e1d9b78de4577dce511 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3d48142371674e1d9b78de4577dce511 2024-11-15T00:45:44,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/aafa51c8a9ae41ec8c20cc93f7885a2d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/aafa51c8a9ae41ec8c20cc93f7885a2d 2024-11-15T00:45:44,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8332d3a26e11420eaba47f005c170db6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8332d3a26e11420eaba47f005c170db6 2024-11-15T00:45:44,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/4bf24f3cea314b60aad4aad7d6e4c88a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/4bf24f3cea314b60aad4aad7d6e4c88a 2024-11-15T00:45:44,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/52ef6f7f4685488cbcfaecd6bbb67cfd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/52ef6f7f4685488cbcfaecd6bbb67cfd 2024-11-15T00:45:44,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/56f9cc5fa524496bb9203fb0c9f96cbd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/56f9cc5fa524496bb9203fb0c9f96cbd 2024-11-15T00:45:44,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/dc218d0517434b51a28bd04ef445b2d7 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/dc218d0517434b51a28bd04ef445b2d7 2024-11-15T00:45:44,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3adda042f3f942bc95107dba0189d478 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/3adda042f3f942bc95107dba0189d478 2024-11-15T00:45:44,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/c50837345e0d41489d5af68857f7e5d8 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/c50837345e0d41489d5af68857f7e5d8 2024-11-15T00:45:44,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/e3dc4345f6e0448eadceb53f54a64b67 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/e3dc4345f6e0448eadceb53f54a64b67 2024-11-15T00:45:44,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8a20b3f6efac4d58aeb58918c95f49be to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/8a20b3f6efac4d58aeb58918c95f49be 2024-11-15T00:45:44,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/5919c316a6ac421fb8240016608f5c77 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/5919c316a6ac421fb8240016608f5c77 2024-11-15T00:45:44,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/b95514fe2f5a4d2f9f1c0ceebe05c499 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/b95514fe2f5a4d2f9f1c0ceebe05c499 2024-11-15T00:45:44,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/a285e42d75fe4ec781282e5d6beac63f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/a285e42d75fe4ec781282e5d6beac63f 2024-11-15T00:45:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:45:44,650 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=4 2024-11-15T00:45:44,651 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. 2024-11-15T00:45:44,651 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for f762865d4d0d37fcd928550c43e0b373: Waiting for close lock at 1731631544172Running coprocessor pre-close hooks at 1731631544172Disabling compacts and flushes for region at 1731631544172Disabling writes for close at 1731631544172Obtaining lock to block concurrent updates at 1731631544172Preparing flush snapshotting stores in f762865d4d0d37fcd928550c43e0b373 at 1731631544172Finished memstore snapshotting TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731631544173 (+1 ms)Flushing stores of TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373. at 1731631544173Flushing f762865d4d0d37fcd928550c43e0b373/A: creating writer at 1731631544173Flushing mob file f762865d4d0d37fcd928550c43e0b373/A: appending metadata at 1731631544184 (+11 ms)Flushing mob file f762865d4d0d37fcd928550c43e0b373/A: closing flushed file at 1731631544184Flushing f762865d4d0d37fcd928550c43e0b373/A: appending metadata at 1731631544228 (+44 ms)Flushing f762865d4d0d37fcd928550c43e0b373/A: closing flushed file at 1731631544228Flushing f762865d4d0d37fcd928550c43e0b373/B: creating writer at 1731631544251 (+23 ms)Flushing f762865d4d0d37fcd928550c43e0b373/B: appending metadata at 1731631544256 (+5 ms)Flushing f762865d4d0d37fcd928550c43e0b373/B: closing flushed file at 1731631544256Flushing f762865d4d0d37fcd928550c43e0b373/C: creating writer at 1731631544283 (+27 ms)Flushing f762865d4d0d37fcd928550c43e0b373/C: appending metadata at 1731631544293 (+10 ms)Flushing f762865d4d0d37fcd928550c43e0b373/C: closing flushed file at 1731631544293Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e4e5c09: reopening flushed file at 1731631544327 (+34 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@277969fb: reopening flushed file at 1731631544344 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4be7dd99: reopening flushed file at 1731631544353 (+9 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f762865d4d0d37fcd928550c43e0b373 in 208ms, sequenceid=334, compaction requested=true at 1731631544380 (+27 ms)Writing region close event to WAL at 1731631544647 (+267 ms)Running coprocessor post-close hooks at 1731631544651 (+4 ms)Closed at 1731631544651 2024-11-15T00:45:44,654 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:44,654 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=f762865d4d0d37fcd928550c43e0b373, regionState=CLOSED 2024-11-15T00:45:44,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:45:44,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T00:45:44,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure f762865d4d0d37fcd928550c43e0b373, server=fb2f6cb7846a,34537,1731631472208 in 637 msec 2024-11-15T00:45:44,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-15T00:45:44,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f762865d4d0d37fcd928550c43e0b373, UNASSIGN in 644 msec 2024-11-15T00:45:44,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T00:45:44,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 647 msec 2024-11-15T00:45:44,662 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631544662"}]},"ts":"1731631544662"} 2024-11-15T00:45:44,663 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:45:44,663 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:45:44,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 659 msec 2024-11-15T00:45:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:45:45,161 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:45,165 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:45:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:45,173 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T00:45:45,174 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:45,186 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,197 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/recovered.edits] 2024-11-15T00:45:45,202 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/5f3600772a7c4d0da13e8bc1335781f0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/5f3600772a7c4d0da13e8bc1335781f0 2024-11-15T00:45:45,204 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/684e1908ecc3478893cb135dc7bd3283 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/A/684e1908ecc3478893cb135dc7bd3283 2024-11-15T00:45:45,207 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2c8733e39fb747b6b1b56e6ce579d713 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/2c8733e39fb747b6b1b56e6ce579d713 2024-11-15T00:45:45,208 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/a14a01e5265f4441a26b77f3dad61daf to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/B/a14a01e5265f4441a26b77f3dad61daf 2024-11-15T00:45:45,215 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/9875822bbe3e42ff811e7683533cc45e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/9875822bbe3e42ff811e7683533cc45e 2024-11-15T00:45:45,216 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/cb2aa0a63f9c4f2384bea7b348fff023 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/cb2aa0a63f9c4f2384bea7b348fff023 2024-11-15T00:45:45,218 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/dbdfb23a52e141d28f8c32dc7c303378 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/C/dbdfb23a52e141d28f8c32dc7c303378 2024-11-15T00:45:45,221 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/recovered.edits/337.seqid to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373/recovered.edits/337.seqid 2024-11-15T00:45:45,222 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,222 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:45:45,223 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:45:45,226 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T00:45:45,236 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111506490d83b40e4c4d93cc71f25ab3fc88_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111506490d83b40e4c4d93cc71f25ab3fc88_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,239 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150a196d33aa8a4b169fcd9aa29e433c37_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150a196d33aa8a4b169fcd9aa29e433c37_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,241 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115184c07f1a16f4ff2b65f96bd42f0599c_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115184c07f1a16f4ff2b65f96bd42f0599c_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,242 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151bde479d1261469d888c63485f29000e_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151bde479d1261469d888c63485f29000e_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,244 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151eed5a89c731439b93d88e21043d97c7_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151eed5a89c731439b93d88e21043d97c7_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,245 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152e211dfa01d14c44b931aa63dc41359f_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152e211dfa01d14c44b931aa63dc41359f_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,247 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f47519a4a6043a6b4a837f4b3c29085_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f47519a4a6043a6b4a837f4b3c29085_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,249 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f8b8fe69b3248649548954f130c3796_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f8b8fe69b3248649548954f130c3796_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,250 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111545221760f1b94fe8a748e7ffb6301f18_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111545221760f1b94fe8a748e7ffb6301f18_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,252 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155e705829d469487883542458d11c0055_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155e705829d469487883542458d11c0055_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,256 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569d356a0bbae485893c94b42fc5db9a2_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569d356a0bbae485893c94b42fc5db9a2_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,258 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115714fd1768ec844c8b36000c9508a90c0_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115714fd1768ec844c8b36000c9508a90c0_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,259 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111572c1c2291761423cbb7a3e485e230f07_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111572c1c2291761423cbb7a3e485e230f07_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,260 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111573ef4772c86e4a77b2889125224b8260_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111573ef4772c86e4a77b2889125224b8260_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,261 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111579d4b99af67643babb62b6a75b095d50_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111579d4b99af67643babb62b6a75b095d50_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,263 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158a2c19b2e7854ba9a47ff2fb7c774012_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158a2c19b2e7854ba9a47ff2fb7c774012_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,264 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b6722f89d910488b9c41884d7f1a3d67_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b6722f89d910488b9c41884d7f1a3d67_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,265 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bc197e2baf9945aca4e6aa8822ad5b76_f762865d4d0d37fcd928550c43e0b373 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bc197e2baf9945aca4e6aa8822ad5b76_f762865d4d0d37fcd928550c43e0b373 2024-11-15T00:45:45,266 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:45:45,273 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:45,276 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:45:45,279 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:45:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T00:45:45,281 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:45,282 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:45:45,282 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631545282"}]},"ts":"9223372036854775807"} 2024-11-15T00:45:45,284 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:45:45,284 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f762865d4d0d37fcd928550c43e0b373, NAME => 'TestAcidGuarantees,,1731631510805.f762865d4d0d37fcd928550c43e0b373.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:45:45,284 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:45:45,285 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631545285"}]},"ts":"9223372036854775807"} 2024-11-15T00:45:45,289 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:45:45,290 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:45,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 125 msec 2024-11-15T00:45:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T00:45:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:45:45,490 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:45,500 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/cluster_c94eddf5-b65f-7607-3f6e-7f317580d804/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1114251894_22 at /127.0.0.1:37572 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/cluster_c94eddf5-b65f-7607-3f6e-7f317580d804/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1022 (was 894) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=960 (was 1982) 2024-11-15T00:45:45,509 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=1022, ProcessCount=11, AvailableMemoryMB=959 2024-11-15T00:45:45,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:45:45,510 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:45:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:45:45,512 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:45:45,512 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:45,512 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-15T00:45:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T00:45:45,513 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:45:45,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742030_1206 (size=1061) 2024-11-15T00:45:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T00:45:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T00:45:45,921 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403 2024-11-15T00:45:45,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742031_1207 (size=53) 2024-11-15T00:45:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T00:45:46,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:45:46,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 12c7a62476f136ecef48b9c405ac1dfb, disabling compactions & flushes 2024-11-15T00:45:46,328 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:46,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:46,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. after waiting 0 ms 2024-11-15T00:45:46,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:46,328 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:46,328 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 12c7a62476f136ecef48b9c405ac1dfb: Waiting for close lock at 1731631546328Disabling compacts and flushes for region at 1731631546328Disabling writes for close at 1731631546328Writing region close event to WAL at 1731631546328Closed at 1731631546328 2024-11-15T00:45:46,330 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:45:46,330 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631546330"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631546330"}]},"ts":"1731631546330"} 2024-11-15T00:45:46,332 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:45:46,333 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:45:46,333 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631546333"}]},"ts":"1731631546333"} 2024-11-15T00:45:46,335 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:45:46,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12c7a62476f136ecef48b9c405ac1dfb, ASSIGN}] 2024-11-15T00:45:46,336 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12c7a62476f136ecef48b9c405ac1dfb, ASSIGN 2024-11-15T00:45:46,338 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12c7a62476f136ecef48b9c405ac1dfb, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,34537,1731631472208; forceNewPlan=false, retain=false 2024-11-15T00:45:46,489 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=12c7a62476f136ecef48b9c405ac1dfb, regionState=OPENING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:46,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12c7a62476f136ecef48b9c405ac1dfb, ASSIGN because future has completed 2024-11-15T00:45:46,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:45:46,647 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:46,647 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:45:46,648 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,648 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:45:46,648 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,648 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T00:45:46,650 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,651 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:45:46,652 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12c7a62476f136ecef48b9c405ac1dfb columnFamilyName A 2024-11-15T00:45:46,652 DEBUG [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:46,653 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] regionserver.HStore(327): Store=12c7a62476f136ecef48b9c405ac1dfb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:45:46,653 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,654 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:45:46,654 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12c7a62476f136ecef48b9c405ac1dfb columnFamilyName B 2024-11-15T00:45:46,655 DEBUG [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:46,655 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] regionserver.HStore(327): Store=12c7a62476f136ecef48b9c405ac1dfb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:45:46,655 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,656 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:45:46,656 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12c7a62476f136ecef48b9c405ac1dfb columnFamilyName C 2024-11-15T00:45:46,656 DEBUG [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:45:46,657 INFO [StoreOpener-12c7a62476f136ecef48b9c405ac1dfb-1 {}] regionserver.HStore(327): Store=12c7a62476f136ecef48b9c405ac1dfb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:45:46,657 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:46,657 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,658 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,658 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,659 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,659 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,660 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:45:46,662 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,664 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:45:46,665 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 12c7a62476f136ecef48b9c405ac1dfb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74454824, jitterRate=0.10946333408355713}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:45:46,665 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:46,665 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 12c7a62476f136ecef48b9c405ac1dfb: Running coprocessor pre-open hook at 1731631546648Writing region info on filesystem at 1731631546648Initializing all the Stores at 1731631546649 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631546649Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631546649Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631546650 (+1 ms)Cleaning up temporary data from old regions at 1731631546659 (+9 ms)Running coprocessor post-open hooks at 1731631546665 (+6 ms)Region opened successfully at 1731631546665 2024-11-15T00:45:46,666 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., pid=57, masterSystemTime=1731631546643 2024-11-15T00:45:46,668 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:46,668 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:46,668 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=12c7a62476f136ecef48b9c405ac1dfb, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:46,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:45:46,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-15T00:45:46,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 in 179 msec 2024-11-15T00:45:46,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-15T00:45:46,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12c7a62476f136ecef48b9c405ac1dfb, ASSIGN in 337 msec 2024-11-15T00:45:46,674 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:45:46,674 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631546674"}]},"ts":"1731631546674"} 2024-11-15T00:45:46,676 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:45:46,677 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:45:46,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-15T00:45:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-15T00:45:47,660 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:47,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c11f0a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,662 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c9dc7a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,664 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@645ec3db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,669 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:45:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,33521,1731631471305 2024-11-15T00:45:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56d9129d 2024-11-15T00:45:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:45:47,671 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:45:47,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25e579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,675 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c611307, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,677 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b0c7c29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,682 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424fe29d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,684 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10577570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,689 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cdcd764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,691 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ebced6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,694 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef2bf2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,695 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21461676, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,698 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a82361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,699 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7aebec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@664fbe4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b068bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3087302d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,707 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3718c1fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,709 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab69000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,710 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e53826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,718 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e78fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,719 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5176bd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:45:47,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:45:47,721 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:45:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:45:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:45:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6540aff9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:45:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:45:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:45:47,723 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:45:47,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50b8787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:45:47,725 DEBUG [Thread-997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,725 DEBUG [Thread-995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,729 DEBUG [Thread-1010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,730 DEBUG [Thread-1008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:45:47,730 DEBUG [Thread-991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:45:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:45:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:45:47,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:45:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:47,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T00:45:47,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:47,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:47,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:47,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:47,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:47,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:47,748 DEBUG [Thread-1002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:45:47,760 DEBUG [Thread-993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,761 DEBUG [Thread-999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,762 DEBUG [Thread-1006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:45:47,766 DEBUG [Thread-1004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:45:47,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:45:47,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:45:47,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:47,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:45:47,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:45:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-15T00:45:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:45:47,781 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:45:47,782 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:47,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:45:47,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/72cf5c1a006c442cb852c251298c386d is 50, key is test_row_0/A:col10/1731631547745/Put/seqid=0 2024-11-15T00:45:47,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631547810, max timestamp=1731631547823Num uniques 859; 2024-11-15T00:45:47,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:45:47,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631547810, max timestamp=1731631547823Num uniques 859; 2024-11-15T00:45:47,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631547810, max timestamp=1731631547823Num uniques 859; 2024-11-15T00:45:47,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:45:47,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:45:47,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742032_1208 (size=12001) 2024-11-15T00:45:47,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/72cf5c1a006c442cb852c251298c386d 2024-11-15T00:45:47,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631607856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:47,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631607864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:47,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631607865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:47,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631607866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:47,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631607866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:47,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:47,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/8f1ffe02a5984bbc8abecc16902081d9 is 50, key is test_row_0/B:col10/1731631547745/Put/seqid=0 2024-11-15T00:45:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:45:47,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:47,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:47,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742033_1209 (size=12001) 2024-11-15T00:45:47,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631607969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:47,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631607969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:47,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631607979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:47,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631607979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631607980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:48,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:48,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:45:48,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631608180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631608180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631608189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:48,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631608189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631608189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:48,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:48,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/8f1ffe02a5984bbc8abecc16902081d9 2024-11-15T00:45:48,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/72094f11f9934b9d871eee6fe9292108 is 50, key is test_row_0/C:col10/1731631547745/Put/seqid=0 2024-11-15T00:45:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:45:48,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742034_1210 (size=12001) 2024-11-15T00:45:48,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631608489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:48,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631608498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631608500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631608500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631608500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:48,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:48,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:48,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:48,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:48,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/72094f11f9934b9d871eee6fe9292108 2024-11-15T00:45:48,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/72cf5c1a006c442cb852c251298c386d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/72cf5c1a006c442cb852c251298c386d 2024-11-15T00:45:48,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:48,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:48,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:48,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:48,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/72cf5c1a006c442cb852c251298c386d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T00:45:48,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/8f1ffe02a5984bbc8abecc16902081d9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/8f1ffe02a5984bbc8abecc16902081d9 2024-11-15T00:45:48,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/8f1ffe02a5984bbc8abecc16902081d9, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T00:45:48,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/72094f11f9934b9d871eee6fe9292108 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/72094f11f9934b9d871eee6fe9292108 2024-11-15T00:45:48,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/72094f11f9934b9d871eee6fe9292108, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T00:45:48,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 12c7a62476f136ecef48b9c405ac1dfb in 1149ms, sequenceid=12, compaction requested=false 2024-11-15T00:45:48,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:45:49,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T00:45:49,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:49,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:49,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:49,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:49,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:49,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:49,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/e5a8b31636d74f9eac98dc56f7555b22 is 50, key is test_row_0/A:col10/1731631547863/Put/seqid=0 2024-11-15T00:45:49,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:49,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:49,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742035_1211 (size=14341) 2024-11-15T00:45:49,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/e5a8b31636d74f9eac98dc56f7555b22 2024-11-15T00:45:49,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631609020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631609029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631609031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/048fd3ee4ca2423cb39b32f9947c9325 is 50, key is test_row_0/B:col10/1731631547863/Put/seqid=0 2024-11-15T00:45:49,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631609034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631609035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742036_1212 (size=12001) 2024-11-15T00:45:49,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631609139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631609139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631609149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631609149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631609149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:49,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:49,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:49,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:49,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631609349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631609349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631609359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631609359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631609359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:49,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:49,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/048fd3ee4ca2423cb39b32f9947c9325 2024-11-15T00:45:49,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/840becc3f65a4a5ab2ad271529748b6c is 50, key is test_row_0/C:col10/1731631547863/Put/seqid=0 2024-11-15T00:45:49,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742037_1213 (size=12001) 2024-11-15T00:45:49,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:49,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631609661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631609664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631609669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631609669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631609672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:49,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:49,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:49,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/840becc3f65a4a5ab2ad271529748b6c 2024-11-15T00:45:49,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/e5a8b31636d74f9eac98dc56f7555b22 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/e5a8b31636d74f9eac98dc56f7555b22 2024-11-15T00:45:49,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/e5a8b31636d74f9eac98dc56f7555b22, entries=200, sequenceid=48, filesize=14.0 K 2024-11-15T00:45:49,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/048fd3ee4ca2423cb39b32f9947c9325 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/048fd3ee4ca2423cb39b32f9947c9325 2024-11-15T00:45:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:45:49,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:49,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:49,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:49,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:49,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/048fd3ee4ca2423cb39b32f9947c9325, entries=150, sequenceid=48, filesize=11.7 K 2024-11-15T00:45:49,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/840becc3f65a4a5ab2ad271529748b6c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/840becc3f65a4a5ab2ad271529748b6c 2024-11-15T00:45:49,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/840becc3f65a4a5ab2ad271529748b6c, entries=150, sequenceid=48, filesize=11.7 K 2024-11-15T00:45:49,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 12c7a62476f136ecef48b9c405ac1dfb in 972ms, sequenceid=48, compaction requested=false 2024-11-15T00:45:49,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:50,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T00:45:50,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:50,105 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T00:45:50,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:50,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:50,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:50,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:50,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:50,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/20c3fd06923d4e98b79b032335c6d202 is 50, key is test_row_0/A:col10/1731631549029/Put/seqid=0 2024-11-15T00:45:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:50,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:50,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742038_1214 (size=9657) 2024-11-15T00:45:50,207 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:45:50,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631550213, max timestamp=1731631550235Num uniques 859; 2024-11-15T00:45:50,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:50,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631550213, max timestamp=1731631550236Num uniques 859; 2024-11-15T00:45:50,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:50,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631550213, max timestamp=1731631550235Num uniques 859; 2024-11-15T00:45:50,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:50,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631610269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631610270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631610271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631610278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631610281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631610389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631610389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631610389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631610389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631610399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,585 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/20c3fd06923d4e98b79b032335c6d202 2024-11-15T00:45:50,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/417581b7db084db1877e41a8acbecfe6 is 50, key is test_row_0/B:col10/1731631549029/Put/seqid=0 2024-11-15T00:45:50,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631610599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631610599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742039_1215 (size=9657) 2024-11-15T00:45:50,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631610610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631610610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631610610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631610909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631610909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631610920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631610920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:50,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631610920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:51,011 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/417581b7db084db1877e41a8acbecfe6 2024-11-15T00:45:51,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3c1c96e07e8a4fa08f943b35e93914a6 is 50, key is test_row_0/C:col10/1731631549029/Put/seqid=0 2024-11-15T00:45:51,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742040_1216 (size=9657) 2024-11-15T00:45:51,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631611419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:51,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631611419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:51,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631611429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:51,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631611429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:51,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631611429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:51,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3c1c96e07e8a4fa08f943b35e93914a6 2024-11-15T00:45:51,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/20c3fd06923d4e98b79b032335c6d202 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/20c3fd06923d4e98b79b032335c6d202 2024-11-15T00:45:51,488 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/20c3fd06923d4e98b79b032335c6d202, entries=100, sequenceid=58, filesize=9.4 K 2024-11-15T00:45:51,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/417581b7db084db1877e41a8acbecfe6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/417581b7db084db1877e41a8acbecfe6 2024-11-15T00:45:51,499 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/417581b7db084db1877e41a8acbecfe6, entries=100, sequenceid=58, filesize=9.4 K 2024-11-15T00:45:51,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3c1c96e07e8a4fa08f943b35e93914a6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3c1c96e07e8a4fa08f943b35e93914a6 2024-11-15T00:45:51,513 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3c1c96e07e8a4fa08f943b35e93914a6, entries=100, sequenceid=58, filesize=9.4 K 2024-11-15T00:45:51,514 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 12c7a62476f136ecef48b9c405ac1dfb in 1409ms, sequenceid=58, compaction requested=true 2024-11-15T00:45:51,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:51,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:51,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-15T00:45:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-15T00:45:51,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T00:45:51,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7340 sec 2024-11-15T00:45:51,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 3.7420 sec 2024-11-15T00:45:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:45:51,965 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:51,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-15T00:45:51,974 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:51,975 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:51,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T00:45:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T00:45:52,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T00:45:52,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:52,137 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-15T00:45:52,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:52,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:52,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:52,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:52,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:52,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:52,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/bbcf6d1bad9e451eb5d8b9625d4e5d08 is 50, key is test_row_0/A:col10/1731631550275/Put/seqid=0 2024-11-15T00:45:52,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742041_1217 (size=12001) 2024-11-15T00:45:52,184 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/bbcf6d1bad9e451eb5d8b9625d4e5d08 2024-11-15T00:45:52,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/ec4c4b13cfdc4df2b235fb3e539737e6 is 50, key is test_row_0/B:col10/1731631550275/Put/seqid=0 2024-11-15T00:45:52,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742042_1218 (size=12001) 2024-11-15T00:45:52,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/ec4c4b13cfdc4df2b235fb3e539737e6 2024-11-15T00:45:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/66ba26dd15954abfb1a629acff321933 is 50, key is test_row_0/C:col10/1731631550275/Put/seqid=0 2024-11-15T00:45:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T00:45:52,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742043_1219 (size=12001) 2024-11-15T00:45:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:52,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:52,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631612450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,464 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in 12c7a62476f136ecef48b9c405ac1dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631612458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631612462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631612463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631612464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631612579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631612580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631612580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631612580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T00:45:52,698 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/66ba26dd15954abfb1a629acff321933 2024-11-15T00:45:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/bbcf6d1bad9e451eb5d8b9625d4e5d08 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/bbcf6d1bad9e451eb5d8b9625d4e5d08 2024-11-15T00:45:52,717 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/bbcf6d1bad9e451eb5d8b9625d4e5d08, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T00:45:52,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/ec4c4b13cfdc4df2b235fb3e539737e6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ec4c4b13cfdc4df2b235fb3e539737e6 2024-11-15T00:45:52,725 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ec4c4b13cfdc4df2b235fb3e539737e6, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T00:45:52,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/66ba26dd15954abfb1a629acff321933 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/66ba26dd15954abfb1a629acff321933 2024-11-15T00:45:52,741 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/66ba26dd15954abfb1a629acff321933, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T00:45:52,742 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 12c7a62476f136ecef48b9c405ac1dfb in 605ms, sequenceid=93, compaction requested=true 2024-11-15T00:45:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-15T00:45:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-15T00:45:52,754 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T00:45:52,754 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 772 msec 2024-11-15T00:45:52,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 793 msec 2024-11-15T00:45:52,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T00:45:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:52,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/0aa35c1048574ac9bfa6e430d87bc1f1 is 50, key is test_row_1/A:col10/1731631552457/Put/seqid=0 2024-11-15T00:45:52,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742044_1220 (size=9657) 2024-11-15T00:45:52,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/0aa35c1048574ac9bfa6e430d87bc1f1 2024-11-15T00:45:52,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631552837, max timestamp=1731631552889Num uniques 859; 2024-11-15T00:45:52,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:52,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631552837, max timestamp=1731631552891Num uniques 859; 2024-11-15T00:45:52,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:52,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/32fc602af1d646ee81ecb23b0990ea48 is 50, key is test_row_1/B:col10/1731631552457/Put/seqid=0 2024-11-15T00:45:52,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631552837, max timestamp=1731631552889Num uniques 859; 2024-11-15T00:45:52,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:52,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742045_1221 (size=9657) 2024-11-15T00:45:52,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/32fc602af1d646ee81ecb23b0990ea48 2024-11-15T00:45:52,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631612941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631612946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631612953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631612955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:52,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/f1349f6bfd184a74b20048cf0c5bcd32 is 50, key is test_row_1/C:col10/1731631552457/Put/seqid=0 2024-11-15T00:45:53,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742046_1222 (size=9657) 2024-11-15T00:45:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631613059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631613070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631613070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631613072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T00:45:53,109 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:53,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-15T00:45:53,113 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T00:45:53,114 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:53,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T00:45:53,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T00:45:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631613269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631613289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631613290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631613290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T00:45:53,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:53,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T00:45:53,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/f1349f6bfd184a74b20048cf0c5bcd32 2024-11-15T00:45:53,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/0aa35c1048574ac9bfa6e430d87bc1f1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0aa35c1048574ac9bfa6e430d87bc1f1 2024-11-15T00:45:53,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0aa35c1048574ac9bfa6e430d87bc1f1, entries=100, sequenceid=104, filesize=9.4 K 2024-11-15T00:45:53,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/32fc602af1d646ee81ecb23b0990ea48 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/32fc602af1d646ee81ecb23b0990ea48 2024-11-15T00:45:53,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/32fc602af1d646ee81ecb23b0990ea48, entries=100, sequenceid=104, filesize=9.4 K 2024-11-15T00:45:53,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/f1349f6bfd184a74b20048cf0c5bcd32 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f1349f6bfd184a74b20048cf0c5bcd32 2024-11-15T00:45:53,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f1349f6bfd184a74b20048cf0c5bcd32, entries=100, sequenceid=104, filesize=9.4 K 2024-11-15T00:45:53,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 12c7a62476f136ecef48b9c405ac1dfb in 747ms, sequenceid=104, compaction requested=true 2024-11-15T00:45:53,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:53,540 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:45:53,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:53,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:53,541 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:45:53,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:53,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:53,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:53,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:53,542 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57657 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:45:53,542 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/A is initiating minor compaction (all files) 2024-11-15T00:45:53,543 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/A in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,543 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/72cf5c1a006c442cb852c251298c386d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/e5a8b31636d74f9eac98dc56f7555b22, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/20c3fd06923d4e98b79b032335c6d202, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/bbcf6d1bad9e451eb5d8b9625d4e5d08, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0aa35c1048574ac9bfa6e430d87bc1f1] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=56.3 K 2024-11-15T00:45:53,544 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72cf5c1a006c442cb852c251298c386d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731631547745 2024-11-15T00:45:53,544 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 55317 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:45:53,544 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a8b31636d74f9eac98dc56f7555b22, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731631547861 2024-11-15T00:45:53,544 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/B is initiating minor compaction (all files) 2024-11-15T00:45:53,544 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/B in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,544 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/8f1ffe02a5984bbc8abecc16902081d9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/048fd3ee4ca2423cb39b32f9947c9325, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/417581b7db084db1877e41a8acbecfe6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ec4c4b13cfdc4df2b235fb3e539737e6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/32fc602af1d646ee81ecb23b0990ea48] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=54.0 K 2024-11-15T00:45:53,546 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20c3fd06923d4e98b79b032335c6d202, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731631549029 2024-11-15T00:45:53,546 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f1ffe02a5984bbc8abecc16902081d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731631547745 2024-11-15T00:45:53,546 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbcf6d1bad9e451eb5d8b9625d4e5d08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731631550268 2024-11-15T00:45:53,547 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa35c1048574ac9bfa6e430d87bc1f1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731631552457 2024-11-15T00:45:53,547 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 048fd3ee4ca2423cb39b32f9947c9325, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731631547862 2024-11-15T00:45:53,547 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 417581b7db084db1877e41a8acbecfe6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731631549029 2024-11-15T00:45:53,548 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ec4c4b13cfdc4df2b235fb3e539737e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731631550268 2024-11-15T00:45:53,548 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 32fc602af1d646ee81ecb23b0990ea48, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731631552457 2024-11-15T00:45:53,577 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#A#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:53,578 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/7adc91fdeaa846c0a8670bbea5fb6a46 is 50, key is test_row_0/A:col10/1731631550275/Put/seqid=0 2024-11-15T00:45:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:53,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T00:45:53,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-15T00:45:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:53,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:53,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:53,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:53,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,587 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:53,587 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/87a0acc84444459e9600a9ae7467b2c9 is 50, key is test_row_0/B:col10/1731631550275/Put/seqid=0 2024-11-15T00:45:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742047_1223 (size=12173) 2024-11-15T00:45:53,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2b134e9b72764bfaad478968b8fd7e67 is 50, key is test_row_0/A:col10/1731631552952/Put/seqid=0 2024-11-15T00:45:53,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742048_1224 (size=12173) 2024-11-15T00:45:53,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742049_1225 (size=14491) 2024-11-15T00:45:53,633 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/87a0acc84444459e9600a9ae7467b2c9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/87a0acc84444459e9600a9ae7467b2c9 2024-11-15T00:45:53,655 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/B of 12c7a62476f136ecef48b9c405ac1dfb into 87a0acc84444459e9600a9ae7467b2c9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:53,656 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:53,656 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/B, priority=11, startTime=1731631553541; duration=0sec 2024-11-15T00:45:53,656 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:53,656 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:B 2024-11-15T00:45:53,656 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:45:53,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631613618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,658 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 55317 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:45:53,658 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/C is initiating minor compaction (all files) 2024-11-15T00:45:53,658 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/C in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,658 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/72094f11f9934b9d871eee6fe9292108, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/840becc3f65a4a5ab2ad271529748b6c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3c1c96e07e8a4fa08f943b35e93914a6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/66ba26dd15954abfb1a629acff321933, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f1349f6bfd184a74b20048cf0c5bcd32] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=54.0 K 2024-11-15T00:45:53,659 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 72094f11f9934b9d871eee6fe9292108, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731631547745 2024-11-15T00:45:53,659 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 840becc3f65a4a5ab2ad271529748b6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731631547862 2024-11-15T00:45:53,659 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c1c96e07e8a4fa08f943b35e93914a6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731631549029 2024-11-15T00:45:53,659 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 66ba26dd15954abfb1a629acff321933, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731631550268 2024-11-15T00:45:53,660 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f1349f6bfd184a74b20048cf0c5bcd32, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731631552457 2024-11-15T00:45:53,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631613656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631613656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631613657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,674 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#C#compaction#182 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:53,676 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/a34ea9f6db9d4798b5248ac65f212751 is 50, key is test_row_0/C:col10/1731631550275/Put/seqid=0 2024-11-15T00:45:53,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742050_1226 (size=12173) 2024-11-15T00:45:53,724 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/a34ea9f6db9d4798b5248ac65f212751 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/a34ea9f6db9d4798b5248ac65f212751 2024-11-15T00:45:53,736 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/C of 12c7a62476f136ecef48b9c405ac1dfb into a34ea9f6db9d4798b5248ac65f212751(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:53,736 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:53,736 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/C, priority=11, startTime=1731631553542; duration=0sec 2024-11-15T00:45:53,737 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:53,737 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:C 2024-11-15T00:45:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T00:45:53,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T00:45:53,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:53,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631613759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631613769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631613769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631613769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T00:45:53,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:53,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:53,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:53,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631613972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631613979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631613979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631613990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:53,999 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/7adc91fdeaa846c0a8670bbea5fb6a46 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7adc91fdeaa846c0a8670bbea5fb6a46 2024-11-15T00:45:54,007 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/A of 12c7a62476f136ecef48b9c405ac1dfb into 7adc91fdeaa846c0a8670bbea5fb6a46(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:54,007 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:54,007 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/A, priority=11, startTime=1731631553540; duration=0sec 2024-11-15T00:45:54,007 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:54,007 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:A 2024-11-15T00:45:54,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2b134e9b72764bfaad478968b8fd7e67 2024-11-15T00:45:54,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/2793f0db4d2d4913bc663e8a6eaa34ed is 50, key is test_row_0/B:col10/1731631552952/Put/seqid=0 2024-11-15T00:45:54,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742051_1227 (size=12151) 2024-11-15T00:45:54,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/2793f0db4d2d4913bc663e8a6eaa34ed 2024-11-15T00:45:54,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T00:45:54,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:54,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:54,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:54,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:54,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:54,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/0627126cc6784d8bb9a073ec1839d2a5 is 50, key is test_row_0/C:col10/1731631552952/Put/seqid=0 2024-11-15T00:45:54,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742052_1228 (size=12151) 2024-11-15T00:45:54,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T00:45:54,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:54,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:54,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:54,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:54,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T00:45:54,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631614279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631614299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631614299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631614300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T00:45:54,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:54,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:54,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:54,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:54,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:54,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/0627126cc6784d8bb9a073ec1839d2a5 2024-11-15T00:45:54,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631614479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2b134e9b72764bfaad478968b8fd7e67 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2b134e9b72764bfaad478968b8fd7e67 2024-11-15T00:45:54,480 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in 12c7a62476f136ecef48b9c405ac1dfb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:45:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2b134e9b72764bfaad478968b8fd7e67, entries=200, sequenceid=139, filesize=14.2 K 2024-11-15T00:45:54,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/2793f0db4d2d4913bc663e8a6eaa34ed as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2793f0db4d2d4913bc663e8a6eaa34ed 2024-11-15T00:45:54,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2793f0db4d2d4913bc663e8a6eaa34ed, entries=150, sequenceid=139, filesize=11.9 K 2024-11-15T00:45:54,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/0627126cc6784d8bb9a073ec1839d2a5 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/0627126cc6784d8bb9a073ec1839d2a5 2024-11-15T00:45:54,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/0627126cc6784d8bb9a073ec1839d2a5, entries=150, sequenceid=139, filesize=11.9 K 2024-11-15T00:45:54,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=60.38 KB/61830 for 12c7a62476f136ecef48b9c405ac1dfb in 922ms, sequenceid=139, compaction requested=false 2024-11-15T00:45:54,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:54,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T00:45:54,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:54,516 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:45:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:54,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/cb7a724fa83946fd81efdee54f772902 is 50, key is test_row_0/A:col10/1731631553616/Put/seqid=0 2024-11-15T00:45:54,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742053_1229 (size=12151) 2024-11-15T00:45:54,539 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/cb7a724fa83946fd81efdee54f772902 2024-11-15T00:45:54,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e5b674a608304c119c35693b93f11505 is 50, key is test_row_0/B:col10/1731631553616/Put/seqid=0 2024-11-15T00:45:54,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742054_1230 (size=12151) 2024-11-15T00:45:54,568 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e5b674a608304c119c35693b93f11505 2024-11-15T00:45:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/836941d8fa1547279f03ee39da822a66 is 50, key is test_row_0/C:col10/1731631553616/Put/seqid=0 2024-11-15T00:45:54,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742055_1231 (size=12151) 2024-11-15T00:45:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:54,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:54,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631614852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631614852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631614853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631614857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631614969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631614969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631614969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631614969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:54,984 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/836941d8fa1547279f03ee39da822a66 2024-11-15T00:45:54,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/cb7a724fa83946fd81efdee54f772902 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb7a724fa83946fd81efdee54f772902 2024-11-15T00:45:54,998 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb7a724fa83946fd81efdee54f772902, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T00:45:55,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e5b674a608304c119c35693b93f11505 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e5b674a608304c119c35693b93f11505 2024-11-15T00:45:55,006 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e5b674a608304c119c35693b93f11505, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T00:45:55,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/836941d8fa1547279f03ee39da822a66 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/836941d8fa1547279f03ee39da822a66 2024-11-15T00:45:55,014 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/836941d8fa1547279f03ee39da822a66, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T00:45:55,016 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 12c7a62476f136ecef48b9c405ac1dfb in 500ms, sequenceid=154, compaction requested=true 2024-11-15T00:45:55,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:55,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:55,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-15T00:45:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-15T00:45:55,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-15T00:45:55,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9030 sec 2024-11-15T00:45:55,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.9100 sec 2024-11-15T00:45:55,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T00:45:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:55,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:55,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/a5bd125137f748c8943cf0b8267b7c11 is 50, key is test_row_0/A:col10/1731631555183/Put/seqid=0 2024-11-15T00:45:55,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742056_1232 (size=16931) 2024-11-15T00:45:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T00:45:55,259 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:55,260 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-15T00:45:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T00:45:55,265 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:55,266 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:55,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:55,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631615263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631615261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631615269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631615270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T00:45:55,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631615379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631615380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631615380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631615390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T00:45:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:55,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T00:45:55,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:55,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:55,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:55,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T00:45:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631615590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631615600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631615600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631615600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/a5bd125137f748c8943cf0b8267b7c11 2024-11-15T00:45:55,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/4ac827467de340c0b79810e595c51959 is 50, key is test_row_0/B:col10/1731631555183/Put/seqid=0 2024-11-15T00:45:55,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742057_1233 (size=12151) 2024-11-15T00:45:55,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T00:45:55,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:55,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:55,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:55,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T00:45:55,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T00:45:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:55,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631615900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631615909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631615909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631615909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:56,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T00:45:56,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:56,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/4ac827467de340c0b79810e595c51959 2024-11-15T00:45:56,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/00de52224554481983f35de2628eff9f is 50, key is test_row_0/C:col10/1731631555183/Put/seqid=0 2024-11-15T00:45:56,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742058_1234 (size=12151) 2024-11-15T00:45:56,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T00:45:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T00:45:56,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:56,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T00:45:56,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631616419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:56,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631616419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:56,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631616419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:56,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631616419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:56,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T00:45:56,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:56,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:56,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/00de52224554481983f35de2628eff9f 2024-11-15T00:45:56,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/a5bd125137f748c8943cf0b8267b7c11 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/a5bd125137f748c8943cf0b8267b7c11 2024-11-15T00:45:56,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/a5bd125137f748c8943cf0b8267b7c11, entries=250, sequenceid=176, filesize=16.5 K 2024-11-15T00:45:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/4ac827467de340c0b79810e595c51959 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/4ac827467de340c0b79810e595c51959 2024-11-15T00:45:56,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/4ac827467de340c0b79810e595c51959, entries=150, sequenceid=176, filesize=11.9 K 2024-11-15T00:45:56,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/00de52224554481983f35de2628eff9f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/00de52224554481983f35de2628eff9f 2024-11-15T00:45:56,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/00de52224554481983f35de2628eff9f, entries=150, sequenceid=176, filesize=11.9 K 2024-11-15T00:45:56,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 12c7a62476f136ecef48b9c405ac1dfb in 1435ms, sequenceid=176, compaction requested=true 2024-11-15T00:45:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:56,620 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:45:56,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:56,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:56,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:56,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:56,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:56,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:45:56,624 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:45:56,627 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55746 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:45:56,627 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/A is initiating minor compaction (all files) 2024-11-15T00:45:56,627 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/A in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,627 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7adc91fdeaa846c0a8670bbea5fb6a46, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2b134e9b72764bfaad478968b8fd7e67, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb7a724fa83946fd81efdee54f772902, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/a5bd125137f748c8943cf0b8267b7c11] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=54.4 K 2024-11-15T00:45:56,628 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 7adc91fdeaa846c0a8670bbea5fb6a46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731631550275 2024-11-15T00:45:56,628 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b134e9b72764bfaad478968b8fd7e67, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731631552870 2024-11-15T00:45:56,629 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting cb7a724fa83946fd81efdee54f772902, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731631553616 2024-11-15T00:45:56,629 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a5bd125137f748c8943cf0b8267b7c11, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731631554842 2024-11-15T00:45:56,633 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48626 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:45:56,633 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/B is initiating minor compaction (all files) 2024-11-15T00:45:56,633 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/B in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,633 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/87a0acc84444459e9600a9ae7467b2c9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2793f0db4d2d4913bc663e8a6eaa34ed, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e5b674a608304c119c35693b93f11505, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/4ac827467de340c0b79810e595c51959] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=47.5 K 2024-11-15T00:45:56,635 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87a0acc84444459e9600a9ae7467b2c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731631550275 2024-11-15T00:45:56,636 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2793f0db4d2d4913bc663e8a6eaa34ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731631552950 2024-11-15T00:45:56,637 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5b674a608304c119c35693b93f11505, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731631553616 2024-11-15T00:45:56,638 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ac827467de340c0b79810e595c51959, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731631554842 2024-11-15T00:45:56,647 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#A#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:56,648 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/6da8156b87154d11a52a117b9f86e7ff is 50, key is test_row_0/A:col10/1731631555183/Put/seqid=0 2024-11-15T00:45:56,655 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:56,655 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/a4e1ade85d994d248b9ae43ce9872c0a is 50, key is test_row_0/B:col10/1731631555183/Put/seqid=0 2024-11-15T00:45:56,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742059_1235 (size=12459) 2024-11-15T00:45:56,668 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/6da8156b87154d11a52a117b9f86e7ff as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6da8156b87154d11a52a117b9f86e7ff 2024-11-15T00:45:56,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T00:45:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,674 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T00:45:56,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:56,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:56,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:56,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:56,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:56,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:56,681 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/A of 12c7a62476f136ecef48b9c405ac1dfb into 6da8156b87154d11a52a117b9f86e7ff(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:56,681 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:56,681 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/A, priority=12, startTime=1731631556620; duration=0sec 2024-11-15T00:45:56,681 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:56,681 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:A 2024-11-15T00:45:56,681 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:45:56,684 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48626 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:45:56,684 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/C is initiating minor compaction (all files) 2024-11-15T00:45:56,685 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/C in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,685 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/a34ea9f6db9d4798b5248ac65f212751, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/0627126cc6784d8bb9a073ec1839d2a5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/836941d8fa1547279f03ee39da822a66, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/00de52224554481983f35de2628eff9f] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=47.5 K 2024-11-15T00:45:56,688 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a34ea9f6db9d4798b5248ac65f212751, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731631550275 2024-11-15T00:45:56,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742060_1236 (size=12459) 2024-11-15T00:45:56,693 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 0627126cc6784d8bb9a073ec1839d2a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731631552950 2024-11-15T00:45:56,696 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 836941d8fa1547279f03ee39da822a66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731631553616 2024-11-15T00:45:56,697 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 00de52224554481983f35de2628eff9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731631554842 2024-11-15T00:45:56,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/eea733d943f6404ba8b4b380f8cd96d6 is 50, key is test_row_1/A:col10/1731631555262/Put/seqid=0 2024-11-15T00:45:56,703 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/a4e1ade85d994d248b9ae43ce9872c0a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a4e1ade85d994d248b9ae43ce9872c0a 2024-11-15T00:45:56,717 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/B of 12c7a62476f136ecef48b9c405ac1dfb into a4e1ade85d994d248b9ae43ce9872c0a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:56,717 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:56,717 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/B, priority=12, startTime=1731631556621; duration=0sec 2024-11-15T00:45:56,717 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:56,717 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:B 2024-11-15T00:45:56,734 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#C#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:56,734 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/6f7304baee444448b277969808e2fdd6 is 50, key is test_row_0/C:col10/1731631555183/Put/seqid=0 2024-11-15T00:45:56,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742061_1237 (size=9757) 2024-11-15T00:45:56,739 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/eea733d943f6404ba8b4b380f8cd96d6 2024-11-15T00:45:56,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e7ddd59204d14634a731c84c4bf3f1ba is 50, key is test_row_1/B:col10/1731631555262/Put/seqid=0 2024-11-15T00:45:56,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742062_1238 (size=12459) 2024-11-15T00:45:56,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742063_1239 (size=9757) 2024-11-15T00:45:56,804 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e7ddd59204d14634a731c84c4bf3f1ba 2024-11-15T00:45:56,810 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/6f7304baee444448b277969808e2fdd6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/6f7304baee444448b277969808e2fdd6 2024-11-15T00:45:56,824 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/C of 12c7a62476f136ecef48b9c405ac1dfb into 6f7304baee444448b277969808e2fdd6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:56,824 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:56,824 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/C, priority=12, startTime=1731631556621; duration=0sec 2024-11-15T00:45:56,824 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:56,824 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:C 2024-11-15T00:45:56,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/b53363151bc940e7ac2df61401455cd6 is 50, key is test_row_1/C:col10/1731631555262/Put/seqid=0 2024-11-15T00:45:56,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742064_1240 (size=9757) 2024-11-15T00:45:56,868 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/b53363151bc940e7ac2df61401455cd6 2024-11-15T00:45:56,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/eea733d943f6404ba8b4b380f8cd96d6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/eea733d943f6404ba8b4b380f8cd96d6 2024-11-15T00:45:56,926 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/eea733d943f6404ba8b4b380f8cd96d6, entries=100, sequenceid=187, filesize=9.5 K 2024-11-15T00:45:56,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e7ddd59204d14634a731c84c4bf3f1ba as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7ddd59204d14634a731c84c4bf3f1ba 2024-11-15T00:45:56,938 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7ddd59204d14634a731c84c4bf3f1ba, entries=100, sequenceid=187, filesize=9.5 K 2024-11-15T00:45:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/b53363151bc940e7ac2df61401455cd6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/b53363151bc940e7ac2df61401455cd6 2024-11-15T00:45:56,950 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/b53363151bc940e7ac2df61401455cd6, entries=100, sequenceid=187, filesize=9.5 K 2024-11-15T00:45:56,953 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 12c7a62476f136ecef48b9c405ac1dfb in 280ms, sequenceid=187, compaction requested=false 2024-11-15T00:45:56,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:56,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:56,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-15T00:45:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-15T00:45:56,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-15T00:45:56,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6900 sec 2024-11-15T00:45:56,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.7010 sec 2024-11-15T00:45:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T00:45:57,413 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:45:57,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:45:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-15T00:45:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:45:57,417 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:45:57,418 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:45:57,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:45:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:57,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T00:45:57,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:57,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:57,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2ada9c6f168243ccb98f6eb327e36722 is 50, key is test_row_0/A:col10/1731631557451/Put/seqid=0 2024-11-15T00:45:57,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742065_1241 (size=16931) 2024-11-15T00:45:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:45:57,529 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631557488, max timestamp=1731631557522Num uniques 859; 2024-11-15T00:45:57,529 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:45:57,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631557488, max timestamp=1731631557522Num uniques 859; 2024-11-15T00:45:57,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:45:57,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2ada9c6f168243ccb98f6eb327e36722 2024-11-15T00:45:57,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631557488, max timestamp=1731631557522Num uniques 859; 2024-11-15T00:45:57,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:45:57,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/5cb7c7d1ebfc446c9ac337ed047a4aeb is 50, key is test_row_0/B:col10/1731631557451/Put/seqid=0 2024-11-15T00:45:57,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:45:57,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:57,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:57,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:57,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:57,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:57,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631617571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631617574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:57,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631617589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:57,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631617589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:57,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742066_1242 (size=12151) 2024-11-15T00:45:57,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/5cb7c7d1ebfc446c9ac337ed047a4aeb 2024-11-15T00:45:57,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/aab220d33f6049acb1c5c5957d00657e is 50, key is test_row_0/C:col10/1731631557451/Put/seqid=0 2024-11-15T00:45:57,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742067_1243 (size=12151) 2024-11-15T00:45:57,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/aab220d33f6049acb1c5c5957d00657e 2024-11-15T00:45:57,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2ada9c6f168243ccb98f6eb327e36722 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2ada9c6f168243ccb98f6eb327e36722 2024-11-15T00:45:57,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2ada9c6f168243ccb98f6eb327e36722, entries=250, sequenceid=200, filesize=16.5 K 2024-11-15T00:45:57,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/5cb7c7d1ebfc446c9ac337ed047a4aeb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/5cb7c7d1ebfc446c9ac337ed047a4aeb 2024-11-15T00:45:57,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/5cb7c7d1ebfc446c9ac337ed047a4aeb, entries=150, sequenceid=200, filesize=11.9 K 2024-11-15T00:45:57,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/aab220d33f6049acb1c5c5957d00657e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/aab220d33f6049acb1c5c5957d00657e 2024-11-15T00:45:57,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/aab220d33f6049acb1c5c5957d00657e, entries=150, sequenceid=200, filesize=11.9 K 2024-11-15T00:45:57,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 12c7a62476f136ecef48b9c405ac1dfb in 254ms, sequenceid=200, compaction requested=true 2024-11-15T00:45:57,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:57,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:57,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:57,708 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:57,708 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:57,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:57,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:57,709 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:57,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:57,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:57,709 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/B is initiating minor compaction (all files) 2024-11-15T00:45:57,709 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/B in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:57,709 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a4e1ade85d994d248b9ae43ce9872c0a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7ddd59204d14634a731c84c4bf3f1ba, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/5cb7c7d1ebfc446c9ac337ed047a4aeb] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=33.6 K 2024-11-15T00:45:57,710 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39147 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:57,710 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/A is initiating minor compaction (all files) 2024-11-15T00:45:57,710 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/A in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:57,710 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6da8156b87154d11a52a117b9f86e7ff, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/eea733d943f6404ba8b4b380f8cd96d6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2ada9c6f168243ccb98f6eb327e36722] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=38.2 K 2024-11-15T00:45:57,710 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a4e1ade85d994d248b9ae43ce9872c0a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731631554842 2024-11-15T00:45:57,710 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6da8156b87154d11a52a117b9f86e7ff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731631554842 2024-11-15T00:45:57,710 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e7ddd59204d14634a731c84c4bf3f1ba, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731631555192 2024-11-15T00:45:57,711 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea733d943f6404ba8b4b380f8cd96d6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731631555192 2024-11-15T00:45:57,711 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cb7c7d1ebfc446c9ac337ed047a4aeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731631557446 2024-11-15T00:45:57,711 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ada9c6f168243ccb98f6eb327e36722, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731631557437 2024-11-15T00:45:57,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731631557563, max timestamp=1731631557714Num uniques 1068; 2024-11-15T00:45:57,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731631557563, max timestamp=1731631557714Num uniques 1068; 2024-11-15T00:45:57,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-15T00:45:57,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-15T00:45:57,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731631557563, max timestamp=1731631557714Num uniques 1068; 2024-11-15T00:45:57,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-15T00:45:57,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=64.85 KB heapSize=174.11 KB 2024-11-15T00:45:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:57,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:57,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:57,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:57,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:57,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:57,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:57,736 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#B#compaction#200 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:57,737 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/eeec8bc65c464694acd078ac3c9682fd is 50, key is test_row_0/B:col10/1731631557451/Put/seqid=0 2024-11-15T00:45:57,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:45:57,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:57,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:57,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:57,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:57,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:45:57,745 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#A#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:57,746 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/f5599cb560ac4713bbe301e5964dd86f is 50, key is test_row_0/A:col10/1731631557451/Put/seqid=0 2024-11-15T00:45:57,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/1cec7252bef7449ba53b40c89b186299 is 50, key is test_row_0/A:col10/1731631557582/Put/seqid=0 2024-11-15T00:45:57,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631557777, max timestamp=1731631557792Num uniques 859; 2024-11-15T00:45:57,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:45:57,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631557777, max timestamp=1731631557792Num uniques 859; 2024-11-15T00:45:57,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:45:57,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631557777, max timestamp=1731631557792Num uniques 859; 2024-11-15T00:45:57,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:45:57,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742068_1244 (size=12561) 2024-11-15T00:45:57,820 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/eeec8bc65c464694acd078ac3c9682fd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/eeec8bc65c464694acd078ac3c9682fd 2024-11-15T00:45:57,828 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/B of 12c7a62476f136ecef48b9c405ac1dfb into eeec8bc65c464694acd078ac3c9682fd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:57,828 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:57,828 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/B, priority=13, startTime=1731631557708; duration=0sec 2024-11-15T00:45:57,828 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:57,828 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:B 2024-11-15T00:45:57,828 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:57,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742069_1245 (size=14541) 2024-11-15T00:45:57,831 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:57,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/1cec7252bef7449ba53b40c89b186299 2024-11-15T00:45:57,831 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/C is initiating minor compaction (all files) 2024-11-15T00:45:57,832 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/C in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:57,832 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/6f7304baee444448b277969808e2fdd6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/b53363151bc940e7ac2df61401455cd6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/aab220d33f6049acb1c5c5957d00657e] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=33.6 K 2024-11-15T00:45:57,832 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f7304baee444448b277969808e2fdd6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731631554842 2024-11-15T00:45:57,833 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting b53363151bc940e7ac2df61401455cd6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731631555192 2024-11-15T00:45:57,833 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting aab220d33f6049acb1c5c5957d00657e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731631557446 2024-11-15T00:45:57,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742070_1246 (size=12561) 2024-11-15T00:45:57,845 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/f5599cb560ac4713bbe301e5964dd86f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f5599cb560ac4713bbe301e5964dd86f 2024-11-15T00:45:57,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e8575b7ffc5f48ecb03c12ba2ad1351b is 50, key is test_row_0/B:col10/1731631557582/Put/seqid=0 2024-11-15T00:45:57,853 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/A of 12c7a62476f136ecef48b9c405ac1dfb into f5599cb560ac4713bbe301e5964dd86f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:57,853 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:57,853 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/A, priority=13, startTime=1731631557707; duration=0sec 2024-11-15T00:45:57,853 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:57,853 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:A 2024-11-15T00:45:57,856 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#C#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:57,857 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/88eb65b8f4604091be49c9d454766911 is 50, key is test_row_0/C:col10/1731631557451/Put/seqid=0 2024-11-15T00:45:57,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742071_1247 (size=12151) 2024-11-15T00:45:57,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742072_1248 (size=12561) 2024-11-15T00:45:57,883 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/88eb65b8f4604091be49c9d454766911 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/88eb65b8f4604091be49c9d454766911 2024-11-15T00:45:57,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731631557836, max timestamp=1731631557862Num uniques 1009; 2024-11-15T00:45:57,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7688, suffix heap size=132904,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-15T00:45:57,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7684, totalHeapSize=17712, min timestamp=1731631557836, max timestamp=1731631557873Num uniques 1009; 2024-11-15T00:45:57,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46209, new segment data size=7684, suffix heap size=132904,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-15T00:45:57,905 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/C of 12c7a62476f136ecef48b9c405ac1dfb into 88eb65b8f4604091be49c9d454766911(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:57,905 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:57,905 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/C, priority=13, startTime=1731631557709; duration=0sec 2024-11-15T00:45:57,905 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:57,905 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:C 2024-11-15T00:45:57,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:45:57,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:57,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:57,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:57,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:57,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:57,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1731631557836, max timestamp=1731631557873Num uniques 1009; 2024-11-15T00:45:57,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46206, new segment data size=7680, suffix heap size=132904,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-15T00:45:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:57,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731631557962, max timestamp=1731631557970Num uniques 1027; 2024-11-15T00:45:57,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-15T00:45:57,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731631557962, max timestamp=1731631557970Num uniques 1027; 2024-11-15T00:45:57,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731631557962, max timestamp=1731631557970Num uniques 1027; 2024-11-15T00:45:57,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-15T00:45:57,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-15T00:45:58,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731631557998, max timestamp=1731631558007Num uniques 1036; 2024-11-15T00:45:58,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731631557998, max timestamp=1731631558007Num uniques 1036; 2024-11-15T00:45:58,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-15T00:45:58,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-15T00:45:58,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731631557998, max timestamp=1731631558007Num uniques 1036; 2024-11-15T00:45:58,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-15T00:45:58,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:45:58,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:45:58,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:58,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:58,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:58,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:58,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:58,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731631558057, max timestamp=1731631558083Num uniques 1095; 2024-11-15T00:45:58,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731631558057, max timestamp=1731631558083Num uniques 1095; 2024-11-15T00:45:58,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-15T00:45:58,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-15T00:45:58,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731631558057, max timestamp=1731631558083Num uniques 1095; 2024-11-15T00:45:58,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-15T00:45:58,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631618121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631618122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1731631558114, max timestamp=1731631558127Num uniques 1154; 2024-11-15T00:45:58,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1731631558114, max timestamp=1731631558127Num uniques 1154; 2024-11-15T00:45:58,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52844, new segment data size=9334, suffix heap size=147984,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-15T00:45:58,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52844, new segment data size=9334, suffix heap size=147984,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-15T00:45:58,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9334, totalHeapSize=21456, min timestamp=1731631558114, max timestamp=1731631558127Num uniques 1154; 2024-11-15T00:45:58,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52844, new segment data size=9334, suffix heap size=147984,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-15T00:45:58,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731631558167, max timestamp=1731631558207Num uniques 1063; 2024-11-15T00:45:58,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731631558167, max timestamp=1731631558207Num uniques 1063; 2024-11-15T00:45:58,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-15T00:45:58,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-15T00:45:58,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731631558167, max timestamp=1731631558207Num uniques 1063; 2024-11-15T00:45:58,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-15T00:45:58,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:45:58,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:58,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:58,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:58,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:58,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:58,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e8575b7ffc5f48ecb03c12ba2ad1351b 2024-11-15T00:45:58,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/2b67267a700e47e5952276182811bc01 is 50, key is test_row_0/C:col10/1731631557582/Put/seqid=0 2024-11-15T00:45:58,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742073_1249 (size=12151) 2024-11-15T00:45:58,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/2b67267a700e47e5952276182811bc01 2024-11-15T00:45:58,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/1cec7252bef7449ba53b40c89b186299 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/1cec7252bef7449ba53b40c89b186299 2024-11-15T00:45:58,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/1cec7252bef7449ba53b40c89b186299, entries=200, sequenceid=238, filesize=14.2 K 2024-11-15T00:45:58,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e8575b7ffc5f48ecb03c12ba2ad1351b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e8575b7ffc5f48ecb03c12ba2ad1351b 2024-11-15T00:45:58,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e8575b7ffc5f48ecb03c12ba2ad1351b, entries=150, sequenceid=238, filesize=11.9 K 2024-11-15T00:45:58,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/2b67267a700e47e5952276182811bc01 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2b67267a700e47e5952276182811bc01 2024-11-15T00:45:58,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2b67267a700e47e5952276182811bc01, entries=150, sequenceid=238, filesize=11.9 K 2024-11-15T00:45:58,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=120.76 KB/123660 for 12c7a62476f136ecef48b9c405ac1dfb in 610ms, sequenceid=238, compaction requested=false 2024-11-15T00:45:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:58,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:58,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=127.47 KB heapSize=364.69 KB 2024-11-15T00:45:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:45:58,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/9ff7c608a6ae420793637b8ae1dab096 is 50, key is test_row_0/A:col10/1731631558331/Put/seqid=0 2024-11-15T00:45:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631618352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631618351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631618354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631618355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742074_1250 (size=12301) 2024-11-15T00:45:58,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/9ff7c608a6ae420793637b8ae1dab096 2024-11-15T00:45:58,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:45:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:58,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:58,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e173f937c25d4910b4b51e3ffeac84cc is 50, key is test_row_0/B:col10/1731631558331/Put/seqid=0 2024-11-15T00:45:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:45:58,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742075_1251 (size=12301) 2024-11-15T00:45:58,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e173f937c25d4910b4b51e3ffeac84cc 2024-11-15T00:45:58,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3e278c9320f34c2ea1ebd9081d4e7878 is 50, key is test_row_0/C:col10/1731631558331/Put/seqid=0 2024-11-15T00:45:58,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742076_1252 (size=12301) 2024-11-15T00:45:58,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3e278c9320f34c2ea1ebd9081d4e7878 2024-11-15T00:45:58,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631618473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631618476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631618479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631618479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/9ff7c608a6ae420793637b8ae1dab096 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/9ff7c608a6ae420793637b8ae1dab096 2024-11-15T00:45:58,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/9ff7c608a6ae420793637b8ae1dab096, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T00:45:58,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e173f937c25d4910b4b51e3ffeac84cc as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e173f937c25d4910b4b51e3ffeac84cc 2024-11-15T00:45:58,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e173f937c25d4910b4b51e3ffeac84cc, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T00:45:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3e278c9320f34c2ea1ebd9081d4e7878 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3e278c9320f34c2ea1ebd9081d4e7878 2024-11-15T00:45:58,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3e278c9320f34c2ea1ebd9081d4e7878, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T00:45:58,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.56 KB/393792, currentSize=53.67 KB/54960 for 12c7a62476f136ecef48b9c405ac1dfb in 212ms, sequenceid=381, compaction requested=true 2024-11-15T00:45:58,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:58,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:45:58,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:58,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:45:58,545 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:58,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:58,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:45:58,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:45:58,545 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:58,546 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:58,546 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/A is initiating minor compaction (all files) 2024-11-15T00:45:58,546 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/A in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:58,547 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f5599cb560ac4713bbe301e5964dd86f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/1cec7252bef7449ba53b40c89b186299, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/9ff7c608a6ae420793637b8ae1dab096] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=38.5 K 2024-11-15T00:45:58,547 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5599cb560ac4713bbe301e5964dd86f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731631557446 2024-11-15T00:45:58,548 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cec7252bef7449ba53b40c89b186299, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731631557563 2024-11-15T00:45:58,548 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ff7c608a6ae420793637b8ae1dab096, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731631558251 2024-11-15T00:45:58,548 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:58,548 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/B is initiating minor compaction (all files) 2024-11-15T00:45:58,549 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/B in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:58,549 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/eeec8bc65c464694acd078ac3c9682fd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e8575b7ffc5f48ecb03c12ba2ad1351b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e173f937c25d4910b4b51e3ffeac84cc] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=36.1 K 2024-11-15T00:45:58,549 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting eeec8bc65c464694acd078ac3c9682fd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731631557446 2024-11-15T00:45:58,550 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e8575b7ffc5f48ecb03c12ba2ad1351b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731631557582 2024-11-15T00:45:58,551 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e173f937c25d4910b4b51e3ffeac84cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731631558251 2024-11-15T00:45:58,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:45:58,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:58,553 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T00:45:58,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:45:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:45:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:45:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:45:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:45:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:45:58,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/7e830d5ab02845a98e6b9a2a568123ca is 50, key is test_row_0/A:col10/1731631558349/Put/seqid=0 2024-11-15T00:45:58,566 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:58,567 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2a0024af6bce4c58ad5c73d972b49263 is 50, key is test_row_0/A:col10/1731631558331/Put/seqid=0 2024-11-15T00:45:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:45:58,580 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#B#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:58,581 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/b7eab77deda349c6b63e936f29467359 is 50, key is test_row_0/B:col10/1731631558331/Put/seqid=0 2024-11-15T00:45:58,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742078_1254 (size=12813) 2024-11-15T00:45:58,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742077_1253 (size=14741) 2024-11-15T00:45:58,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742079_1255 (size=12813) 2024-11-15T00:45:58,657 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/b7eab77deda349c6b63e936f29467359 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/b7eab77deda349c6b63e936f29467359 2024-11-15T00:45:58,664 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2a0024af6bce4c58ad5c73d972b49263 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2a0024af6bce4c58ad5c73d972b49263 2024-11-15T00:45:58,672 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/B of 12c7a62476f136ecef48b9c405ac1dfb into b7eab77deda349c6b63e936f29467359(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:58,672 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:58,672 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/B, priority=13, startTime=1731631558545; duration=0sec 2024-11-15T00:45:58,672 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:45:58,672 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:B 2024-11-15T00:45:58,672 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:45:58,672 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/A of 12c7a62476f136ecef48b9c405ac1dfb into 2a0024af6bce4c58ad5c73d972b49263(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:58,673 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:58,673 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/A, priority=13, startTime=1731631558545; duration=0sec 2024-11-15T00:45:58,673 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:58,673 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:A 2024-11-15T00:45:58,674 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:45:58,674 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/C is initiating minor compaction (all files) 2024-11-15T00:45:58,675 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/C in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:45:58,675 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/88eb65b8f4604091be49c9d454766911, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2b67267a700e47e5952276182811bc01, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3e278c9320f34c2ea1ebd9081d4e7878] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=36.1 K 2024-11-15T00:45:58,676 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 88eb65b8f4604091be49c9d454766911, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731631557446 2024-11-15T00:45:58,677 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b67267a700e47e5952276182811bc01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731631557582 2024-11-15T00:45:58,677 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e278c9320f34c2ea1ebd9081d4e7878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731631558251 2024-11-15T00:45:58,695 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:45:58,695 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/28463d14233943e49f4c40cb2178fe7b is 50, key is test_row_0/C:col10/1731631558331/Put/seqid=0 2024-11-15T00:45:58,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742080_1256 (size=12813) 2024-11-15T00:45:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631618755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631618761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631558721, max timestamp=1731631558773Num uniques 859; 2024-11-15T00:45:58,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:45:58,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631558721, max timestamp=1731631558773Num uniques 859; 2024-11-15T00:45:58,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:45:58,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631558721, max timestamp=1731631558773Num uniques 859; 2024-11-15T00:45:58,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:45:58,789 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/28463d14233943e49f4c40cb2178fe7b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/28463d14233943e49f4c40cb2178fe7b 2024-11-15T00:45:58,811 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/C of 12c7a62476f136ecef48b9c405ac1dfb into 28463d14233943e49f4c40cb2178fe7b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:45:58,811 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:45:58,811 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/C, priority=13, startTime=1731631558545; duration=0sec 2024-11-15T00:45:58,811 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:45:58,811 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:C 2024-11-15T00:45:58,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631618847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631618849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631618856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631618881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631618890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631618970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631618970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:58,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631618970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:58,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:58,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,043 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/7e830d5ab02845a98e6b9a2a568123ca 2024-11-15T00:45:59,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/a9b61758ad524df29e7b7887146e4449 is 50, key is test_row_0/B:col10/1731631558349/Put/seqid=0 2024-11-15T00:45:59,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631619093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631619099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742081_1257 (size=12301) 2024-11-15T00:45:59,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631619179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631619184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631619184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631619412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631619424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631619492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631619499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631619499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,543 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/a9b61758ad524df29e7b7887146e4449 2024-11-15T00:45:59,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/1e6b7ebfca024e57a56dfdc9ef9c0627 is 50, key is test_row_0/C:col10/1731631558349/Put/seqid=0 2024-11-15T00:45:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:45:59,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742082_1258 (size=12301) 2024-11-15T00:45:59,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631619930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:45:59,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:45:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631619939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:45:59,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:45:59,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:00,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631619999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:00,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631620012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:00,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631620012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:00,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:00,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:00,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:00,017 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/1e6b7ebfca024e57a56dfdc9ef9c0627 2024-11-15T00:46:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:00,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/7e830d5ab02845a98e6b9a2a568123ca as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7e830d5ab02845a98e6b9a2a568123ca 2024-11-15T00:46:00,043 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7e830d5ab02845a98e6b9a2a568123ca, entries=200, sequenceid=393, filesize=14.4 K 2024-11-15T00:46:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/a9b61758ad524df29e7b7887146e4449 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a9b61758ad524df29e7b7887146e4449 2024-11-15T00:46:00,053 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a9b61758ad524df29e7b7887146e4449, entries=150, sequenceid=393, filesize=12.0 K 2024-11-15T00:46:00,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/1e6b7ebfca024e57a56dfdc9ef9c0627 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1e6b7ebfca024e57a56dfdc9ef9c0627 2024-11-15T00:46:00,060 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1e6b7ebfca024e57a56dfdc9ef9c0627, entries=150, sequenceid=393, filesize=12.0 K 2024-11-15T00:46:00,061 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=117.73 KB/120560 for 12c7a62476f136ecef48b9c405ac1dfb in 1508ms, sequenceid=393, compaction requested=false 2024-11-15T00:46:00,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:00,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:00,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-15T00:46:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-15T00:46:00,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T00:46:00,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6460 sec 2024-11-15T00:46:00,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.6560 sec 2024-11-15T00:46:00,332 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:46:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:00,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=124.44 KB heapSize=353.47 KB 2024-11-15T00:46:00,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:00,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:00,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:00,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:00,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:00,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:00,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/ceaff70dd87140559ecaaac990c19e50 is 50, key is test_row_0/A:col10/1731631560951/Put/seqid=0 2024-11-15T00:46:00,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742083_1259 (size=12301) 2024-11-15T00:46:00,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/ceaff70dd87140559ecaaac990c19e50 2024-11-15T00:46:00,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e7c2d51074074e3fbe2a9e7d77b15a59 is 50, key is test_row_0/B:col10/1731631560951/Put/seqid=0 2024-11-15T00:46:00,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742084_1260 (size=12301) 2024-11-15T00:46:00,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631620979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:00,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:00,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e7c2d51074074e3fbe2a9e7d77b15a59 2024-11-15T00:46:00,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631620985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:00,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e0f69023ea4049ceb720b1bb6589de40 is 50, key is test_row_0/C:col10/1731631560951/Put/seqid=0 2024-11-15T00:46:01,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631621009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,016 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in 12c7a62476f136ecef48b9c405ac1dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631621029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631621029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,033 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in 12c7a62476f136ecef48b9c405ac1dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,034 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in 12c7a62476f136ecef48b9c405ac1dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742085_1261 (size=12301) 2024-11-15T00:46:01,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631621089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631621099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631621299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631621309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e0f69023ea4049ceb720b1bb6589de40 2024-11-15T00:46:01,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/ceaff70dd87140559ecaaac990c19e50 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/ceaff70dd87140559ecaaac990c19e50 2024-11-15T00:46:01,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/ceaff70dd87140559ecaaac990c19e50, entries=150, sequenceid=431, filesize=12.0 K 2024-11-15T00:46:01,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/e7c2d51074074e3fbe2a9e7d77b15a59 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7c2d51074074e3fbe2a9e7d77b15a59 2024-11-15T00:46:01,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7c2d51074074e3fbe2a9e7d77b15a59, entries=150, sequenceid=431, filesize=12.0 K 2024-11-15T00:46:01,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e0f69023ea4049ceb720b1bb6589de40 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e0f69023ea4049ceb720b1bb6589de40 2024-11-15T00:46:01,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e0f69023ea4049ceb720b1bb6589de40, entries=150, sequenceid=431, filesize=12.0 K 2024-11-15T00:46:01,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~124.44 KB/127430, heapSize ~353.42 KB/361904, currentSize=53.67 KB/54960 for 12c7a62476f136ecef48b9c405ac1dfb in 577ms, sequenceid=431, compaction requested=true 2024-11-15T00:46:01,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:01,531 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:01,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:01,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:01,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:01,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:01,531 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:01,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:01,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:01,532 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:01,532 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/A is initiating minor compaction (all files) 2024-11-15T00:46:01,533 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/A in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:01,533 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2a0024af6bce4c58ad5c73d972b49263, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7e830d5ab02845a98e6b9a2a568123ca, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/ceaff70dd87140559ecaaac990c19e50] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=38.9 K 2024-11-15T00:46:01,534 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a0024af6bce4c58ad5c73d972b49263, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731631558251 2024-11-15T00:46:01,534 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:01,534 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/B is initiating minor compaction (all files) 2024-11-15T00:46:01,534 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e830d5ab02845a98e6b9a2a568123ca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731631558349 2024-11-15T00:46:01,535 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/B in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:01,535 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/b7eab77deda349c6b63e936f29467359, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a9b61758ad524df29e7b7887146e4449, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7c2d51074074e3fbe2a9e7d77b15a59] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=36.5 K 2024-11-15T00:46:01,535 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceaff70dd87140559ecaaac990c19e50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731631558802 2024-11-15T00:46:01,547 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting b7eab77deda349c6b63e936f29467359, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731631558251 2024-11-15T00:46:01,548 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a9b61758ad524df29e7b7887146e4449, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731631558349 2024-11-15T00:46:01,548 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c2d51074074e3fbe2a9e7d77b15a59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731631558802 2024-11-15T00:46:01,558 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#A#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:01,559 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/acec81ae3ae44794be87d82e973e4159 is 50, key is test_row_0/A:col10/1731631560951/Put/seqid=0 2024-11-15T00:46:01,579 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:01,580 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/596bfbf92c144108ade32af160667eca is 50, key is test_row_0/B:col10/1731631560951/Put/seqid=0 2024-11-15T00:46:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:46:01,592 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:01,592 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T00:46:01,595 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:01,596 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:46:01,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:01,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742086_1262 (size=12915) 2024-11-15T00:46:01,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:46:01,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:01,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:01,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:01,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:01,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:01,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:01,626 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/acec81ae3ae44794be87d82e973e4159 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/acec81ae3ae44794be87d82e973e4159 2024-11-15T00:46:01,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742087_1263 (size=12915) 2024-11-15T00:46:01,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/131e077b8679450c90da66c477442c6e is 50, key is test_row_0/A:col10/1731631560970/Put/seqid=0 2024-11-15T00:46:01,646 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/A of 12c7a62476f136ecef48b9c405ac1dfb into acec81ae3ae44794be87d82e973e4159(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:01,646 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:01,646 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/A, priority=13, startTime=1731631561531; duration=0sec 2024-11-15T00:46:01,647 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:01,647 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:A 2024-11-15T00:46:01,647 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:01,649 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/596bfbf92c144108ade32af160667eca as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/596bfbf92c144108ade32af160667eca 2024-11-15T00:46:01,657 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:01,657 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/C is initiating minor compaction (all files) 2024-11-15T00:46:01,657 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/C in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:01,658 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/28463d14233943e49f4c40cb2178fe7b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1e6b7ebfca024e57a56dfdc9ef9c0627, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e0f69023ea4049ceb720b1bb6589de40] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=36.5 K 2024-11-15T00:46:01,659 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28463d14233943e49f4c40cb2178fe7b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731631558251 2024-11-15T00:46:01,660 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e6b7ebfca024e57a56dfdc9ef9c0627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731631558349 2024-11-15T00:46:01,661 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/B of 12c7a62476f136ecef48b9c405ac1dfb into 596bfbf92c144108ade32af160667eca(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:01,661 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:01,661 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/B, priority=13, startTime=1731631561531; duration=0sec 2024-11-15T00:46:01,661 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:01,661 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:B 2024-11-15T00:46:01,661 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0f69023ea4049ceb720b1bb6589de40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731631558802 2024-11-15T00:46:01,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742088_1264 (size=12301) 2024-11-15T00:46:01,679 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:01,680 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e160a3a7b20445c3a79d16fb3e0b746a is 50, key is test_row_0/C:col10/1731631560951/Put/seqid=0 2024-11-15T00:46:01,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/131e077b8679450c90da66c477442c6e 2024-11-15T00:46:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:46:01,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/11111651c2b74e17a09c648ad3bf0297 is 50, key is test_row_0/B:col10/1731631560970/Put/seqid=0 2024-11-15T00:46:01,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742090_1266 (size=12301) 2024-11-15T00:46:01,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/11111651c2b74e17a09c648ad3bf0297 2024-11-15T00:46:01,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T00:46:01,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:01,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:01,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:01,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:01,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:01,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742089_1265 (size=12915) 2024-11-15T00:46:01,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631621763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631621771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,782 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e160a3a7b20445c3a79d16fb3e0b746a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e160a3a7b20445c3a79d16fb3e0b746a 2024-11-15T00:46:01,789 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/C of 12c7a62476f136ecef48b9c405ac1dfb into e160a3a7b20445c3a79d16fb3e0b746a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:01,789 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:01,789 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/C, priority=13, startTime=1731631561531; duration=0sec 2024-11-15T00:46:01,789 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:01,789 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:C 2024-11-15T00:46:01,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3d3d94f5a0c541baa3336f3d3a02be37 is 50, key is test_row_0/C:col10/1731631560970/Put/seqid=0 2024-11-15T00:46:01,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742091_1267 (size=12301) 2024-11-15T00:46:01,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3d3d94f5a0c541baa3336f3d3a02be37 2024-11-15T00:46:01,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/131e077b8679450c90da66c477442c6e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/131e077b8679450c90da66c477442c6e 2024-11-15T00:46:01,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/131e077b8679450c90da66c477442c6e, entries=150, sequenceid=443, filesize=12.0 K 2024-11-15T00:46:01,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/11111651c2b74e17a09c648ad3bf0297 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/11111651c2b74e17a09c648ad3bf0297 2024-11-15T00:46:01,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/11111651c2b74e17a09c648ad3bf0297, entries=150, sequenceid=443, filesize=12.0 K 2024-11-15T00:46:01,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3d3d94f5a0c541baa3336f3d3a02be37 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3d3d94f5a0c541baa3336f3d3a02be37 2024-11-15T00:46:01,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631621879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631621880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:01,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3d3d94f5a0c541baa3336f3d3a02be37, entries=150, sequenceid=443, filesize=12.0 K 2024-11-15T00:46:01,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 12c7a62476f136ecef48b9c405ac1dfb in 266ms, sequenceid=443, compaction requested=false 2024-11-15T00:46:01,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:46:01,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T00:46:01,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:01,916 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:46:01,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:01,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:01,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:01,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:01,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:01,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:01,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/b3256587af43472eb0e7904f5be63ead is 50, key is test_row_0/A:col10/1731631561768/Put/seqid=0 2024-11-15T00:46:01,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742092_1268 (size=12301) 2024-11-15T00:46:01,933 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/b3256587af43472eb0e7904f5be63ead 2024-11-15T00:46:01,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/2b5e9a1cc2b148c1b30efcaf50a4e1c2 is 50, key is test_row_0/B:col10/1731631561768/Put/seqid=0 2024-11-15T00:46:01,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742093_1269 (size=12301) 2024-11-15T00:46:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:02,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:02,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631622166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:02,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:02,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631622173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:46:02,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631622280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:02,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631622289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:02,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:02,385 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/2b5e9a1cc2b148c1b30efcaf50a4e1c2 2024-11-15T00:46:02,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/f07469c97f7b4958a3a41c73f9d741e6 is 50, key is test_row_0/C:col10/1731631561768/Put/seqid=0 2024-11-15T00:46:02,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742094_1270 (size=12301) 2024-11-15T00:46:02,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631622490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:02,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:02,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631622499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:02,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:46:02,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631622800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:02,808 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/f07469c97f7b4958a3a41c73f9d741e6 2024-11-15T00:46:02,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631622809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:02,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/b3256587af43472eb0e7904f5be63ead as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b3256587af43472eb0e7904f5be63ead 2024-11-15T00:46:02,829 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b3256587af43472eb0e7904f5be63ead, entries=150, sequenceid=466, filesize=12.0 K 2024-11-15T00:46:02,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/2b5e9a1cc2b148c1b30efcaf50a4e1c2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2b5e9a1cc2b148c1b30efcaf50a4e1c2 2024-11-15T00:46:02,835 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2b5e9a1cc2b148c1b30efcaf50a4e1c2, entries=150, sequenceid=466, filesize=12.0 K 2024-11-15T00:46:02,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/f07469c97f7b4958a3a41c73f9d741e6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f07469c97f7b4958a3a41c73f9d741e6 2024-11-15T00:46:02,843 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f07469c97f7b4958a3a41c73f9d741e6, entries=150, sequenceid=466, filesize=12.0 K 2024-11-15T00:46:02,845 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 12c7a62476f136ecef48b9c405ac1dfb in 929ms, sequenceid=466, compaction requested=true 2024-11-15T00:46:02,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:02,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:02,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T00:46:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T00:46:02,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T00:46:02,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2540 sec 2024-11-15T00:46:02,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2630 sec 2024-11-15T00:46:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:03,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:46:03,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:03,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:03,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:03,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:03,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:03,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:03,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/f2cd4773eb034a5f8d83da8c7340a348 is 50, key is test_row_0/A:col10/1731631562133/Put/seqid=0 2024-11-15T00:46:03,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742095_1271 (size=12301) 2024-11-15T00:46:03,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631623084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631623091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631623091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631623200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631623209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631623209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631623309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631623319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631623410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631623419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631623419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/f2cd4773eb034a5f8d83da8c7340a348 2024-11-15T00:46:03,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/89ca7d71418c42f78de6d5a9533ae51e is 50, key is test_row_0/B:col10/1731631562133/Put/seqid=0 2024-11-15T00:46:03,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742096_1272 (size=12301) 2024-11-15T00:46:03,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/89ca7d71418c42f78de6d5a9533ae51e 2024-11-15T00:46:03,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/46fb192e44cf49d4b6d593278c74130a is 50, key is test_row_0/C:col10/1731631562133/Put/seqid=0 2024-11-15T00:46:03,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742097_1273 (size=12301) 2024-11-15T00:46:03,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/46fb192e44cf49d4b6d593278c74130a 2024-11-15T00:46:03,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/f2cd4773eb034a5f8d83da8c7340a348 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f2cd4773eb034a5f8d83da8c7340a348 2024-11-15T00:46:03,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f2cd4773eb034a5f8d83da8c7340a348, entries=150, sequenceid=479, filesize=12.0 K 2024-11-15T00:46:03,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/89ca7d71418c42f78de6d5a9533ae51e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/89ca7d71418c42f78de6d5a9533ae51e 2024-11-15T00:46:03,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/89ca7d71418c42f78de6d5a9533ae51e, entries=150, sequenceid=479, filesize=12.0 K 2024-11-15T00:46:03,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/46fb192e44cf49d4b6d593278c74130a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/46fb192e44cf49d4b6d593278c74130a 2024-11-15T00:46:03,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/46fb192e44cf49d4b6d593278c74130a, entries=150, sequenceid=479, filesize=12.0 K 2024-11-15T00:46:03,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 12c7a62476f136ecef48b9c405ac1dfb in 522ms, sequenceid=479, compaction requested=true 2024-11-15T00:46:03,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:03,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:03,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:03,564 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:03,564 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:03,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:03,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:03,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:03,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:03,567 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:03,567 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:03,567 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/A is initiating minor compaction (all files) 2024-11-15T00:46:03,567 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/B is initiating minor compaction (all files) 2024-11-15T00:46:03,567 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/B in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:03,567 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/596bfbf92c144108ade32af160667eca, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/11111651c2b74e17a09c648ad3bf0297, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2b5e9a1cc2b148c1b30efcaf50a4e1c2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/89ca7d71418c42f78de6d5a9533ae51e] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=48.7 K 2024-11-15T00:46:03,568 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/A in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:03,568 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/acec81ae3ae44794be87d82e973e4159, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/131e077b8679450c90da66c477442c6e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b3256587af43472eb0e7904f5be63ead, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f2cd4773eb034a5f8d83da8c7340a348] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=48.7 K 2024-11-15T00:46:03,568 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 596bfbf92c144108ade32af160667eca, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731631558802 2024-11-15T00:46:03,568 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting acec81ae3ae44794be87d82e973e4159, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731631558802 2024-11-15T00:46:03,569 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 11111651c2b74e17a09c648ad3bf0297, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731631560970 2024-11-15T00:46:03,569 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 131e077b8679450c90da66c477442c6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731631560970 2024-11-15T00:46:03,569 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b5e9a1cc2b148c1b30efcaf50a4e1c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731631561739 2024-11-15T00:46:03,569 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3256587af43472eb0e7904f5be63ead, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731631561739 2024-11-15T00:46:03,569 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 89ca7d71418c42f78de6d5a9533ae51e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731631562133 2024-11-15T00:46:03,570 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2cd4773eb034a5f8d83da8c7340a348, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731631562133 2024-11-15T00:46:03,595 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#A#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:03,596 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/062c69a01fd847868be4a666f97d7020 is 50, key is test_row_0/A:col10/1731631562133/Put/seqid=0 2024-11-15T00:46:03,600 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#B#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:03,602 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/f6cd2574f8c14d5eb2959df5404033e6 is 50, key is test_row_0/B:col10/1731631562133/Put/seqid=0 2024-11-15T00:46:03,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742098_1274 (size=13051) 2024-11-15T00:46:03,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742099_1275 (size=13051) 2024-11-15T00:46:03,658 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/f6cd2574f8c14d5eb2959df5404033e6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/f6cd2574f8c14d5eb2959df5404033e6 2024-11-15T00:46:03,667 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/B of 12c7a62476f136ecef48b9c405ac1dfb into f6cd2574f8c14d5eb2959df5404033e6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:03,667 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:03,667 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/B, priority=12, startTime=1731631563564; duration=0sec 2024-11-15T00:46:03,667 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:03,667 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:B 2024-11-15T00:46:03,667 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:03,670 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:03,670 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/C is initiating minor compaction (all files) 2024-11-15T00:46:03,670 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/C in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:03,670 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e160a3a7b20445c3a79d16fb3e0b746a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3d3d94f5a0c541baa3336f3d3a02be37, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f07469c97f7b4958a3a41c73f9d741e6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/46fb192e44cf49d4b6d593278c74130a] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=48.7 K 2024-11-15T00:46:03,671 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e160a3a7b20445c3a79d16fb3e0b746a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731631558802 2024-11-15T00:46:03,671 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d3d94f5a0c541baa3336f3d3a02be37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731631560970 2024-11-15T00:46:03,671 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f07469c97f7b4958a3a41c73f9d741e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731631561739 2024-11-15T00:46:03,672 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 46fb192e44cf49d4b6d593278c74130a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731631562133 2024-11-15T00:46:03,690 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#C#compaction#232 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:03,691 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/feb4457d4ae24e8db7624aac64e424b1 is 50, key is test_row_0/C:col10/1731631562133/Put/seqid=0 2024-11-15T00:46:03,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742100_1276 (size=13051) 2024-11-15T00:46:03,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:46:03,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:03,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:03,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:03,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:03,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:03,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:03,737 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/feb4457d4ae24e8db7624aac64e424b1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/feb4457d4ae24e8db7624aac64e424b1 2024-11-15T00:46:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:46:03,740 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:03,741 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T00:46:03,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/8731251151b5480f99b4e0d5bca31b6c is 50, key is test_row_0/A:col10/1731631563082/Put/seqid=0 2024-11-15T00:46:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T00:46:03,744 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:03,746 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:03,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:03,758 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/C of 12c7a62476f136ecef48b9c405ac1dfb into feb4457d4ae24e8db7624aac64e424b1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:03,758 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:03,758 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/C, priority=12, startTime=1731631563566; duration=0sec 2024-11-15T00:46:03,758 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:03,758 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:C 2024-11-15T00:46:03,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631623761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631623773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631623774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742101_1277 (size=14741) 2024-11-15T00:46:03,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/8731251151b5480f99b4e0d5bca31b6c 2024-11-15T00:46:03,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/0b61a11022f744838be66da94634d1b4 is 50, key is test_row_0/B:col10/1731631563082/Put/seqid=0 2024-11-15T00:46:03,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742102_1278 (size=12301) 2024-11-15T00:46:03,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/0b61a11022f744838be66da94634d1b4 2024-11-15T00:46:03,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/8c7de2a6e4e64c9ba08a794134953525 is 50, key is test_row_0/C:col10/1731631563082/Put/seqid=0 2024-11-15T00:46:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T00:46:03,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742103_1279 (size=12301) 2024-11-15T00:46:03,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/8c7de2a6e4e64c9ba08a794134953525 2024-11-15T00:46:03,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631623880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631623889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631623889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:03,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/8731251151b5480f99b4e0d5bca31b6c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/8731251151b5480f99b4e0d5bca31b6c 2024-11-15T00:46:03,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/8731251151b5480f99b4e0d5bca31b6c, entries=200, sequenceid=500, filesize=14.4 K 2024-11-15T00:46:03,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/0b61a11022f744838be66da94634d1b4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0b61a11022f744838be66da94634d1b4 2024-11-15T00:46:03,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T00:46:03,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:03,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:03,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:03,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:03,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:03,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0b61a11022f744838be66da94634d1b4, entries=150, sequenceid=500, filesize=12.0 K 2024-11-15T00:46:03,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/8c7de2a6e4e64c9ba08a794134953525 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/8c7de2a6e4e64c9ba08a794134953525 2024-11-15T00:46:03,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/8c7de2a6e4e64c9ba08a794134953525, entries=150, sequenceid=500, filesize=12.0 K 2024-11-15T00:46:03,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 12c7a62476f136ecef48b9c405ac1dfb in 207ms, sequenceid=500, compaction requested=false 2024-11-15T00:46:03,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T00:46:04,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T00:46:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:04,068 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:46:04,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:04,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:04,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:04,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:04,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:04,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/4ae8edaff7124373af28e815a26e9722 is 50, key is test_row_0/A:col10/1731631563753/Put/seqid=0 2024-11-15T00:46:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:04,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:04,100 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/062c69a01fd847868be4a666f97d7020 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/062c69a01fd847868be4a666f97d7020 2024-11-15T00:46:04,140 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/A of 12c7a62476f136ecef48b9c405ac1dfb into 062c69a01fd847868be4a666f97d7020(size=12.7 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:04,140 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:04,140 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/A, priority=12, startTime=1731631563564; duration=0sec 2024-11-15T00:46:04,141 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:04,141 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:A 2024-11-15T00:46:04,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742104_1280 (size=12301) 2024-11-15T00:46:04,152 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/4ae8edaff7124373af28e815a26e9722 2024-11-15T00:46:04,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631624195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631624199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/ef0228df312d4e11975383a2a0212a16 is 50, key is test_row_0/B:col10/1731631563753/Put/seqid=0 2024-11-15T00:46:04,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631624206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742105_1281 (size=12301) 2024-11-15T00:46:04,250 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/ef0228df312d4e11975383a2a0212a16 2024-11-15T00:46:04,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/1fbe4e16daa4434da2e5d988c387598f is 50, key is test_row_0/C:col10/1731631563753/Put/seqid=0 2024-11-15T00:46:04,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631624310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631624312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631624319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742106_1282 (size=12301) 2024-11-15T00:46:04,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631624330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631624333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,338 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in 12c7a62476f136ecef48b9c405ac1dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,340 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/1fbe4e16daa4434da2e5d988c387598f 2024-11-15T00:46:04,343 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in 12c7a62476f136ecef48b9c405ac1dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/4ae8edaff7124373af28e815a26e9722 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/4ae8edaff7124373af28e815a26e9722 2024-11-15T00:46:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T00:46:04,394 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/4ae8edaff7124373af28e815a26e9722, entries=150, sequenceid=514, filesize=12.0 K 2024-11-15T00:46:04,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/ef0228df312d4e11975383a2a0212a16 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ef0228df312d4e11975383a2a0212a16 2024-11-15T00:46:04,418 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ef0228df312d4e11975383a2a0212a16, entries=150, sequenceid=514, filesize=12.0 K 2024-11-15T00:46:04,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/1fbe4e16daa4434da2e5d988c387598f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1fbe4e16daa4434da2e5d988c387598f 2024-11-15T00:46:04,443 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1fbe4e16daa4434da2e5d988c387598f, entries=150, sequenceid=514, filesize=12.0 K 2024-11-15T00:46:04,445 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 12c7a62476f136ecef48b9c405ac1dfb in 377ms, sequenceid=514, compaction requested=true 2024-11-15T00:46:04,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:04,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:04,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T00:46:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T00:46:04,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T00:46:04,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 701 msec 2024-11-15T00:46:04,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 712 msec 2024-11-15T00:46:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:04,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:46:04,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:04,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:04,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:04,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:04,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/88b712cdfcb24244a34ee50bccc861ea is 50, key is test_row_0/A:col10/1731631564189/Put/seqid=0 2024-11-15T00:46:04,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742107_1283 (size=12301) 2024-11-15T00:46:04,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631624560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631624566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631624573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631624679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631624679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631624690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T00:46:04,880 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:04,880 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T00:46:04,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:46:04,891 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:04,891 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:04,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:04,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631624889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631624889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:04,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631624900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:04,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:04,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/88b712cdfcb24244a34ee50bccc861ea 2024-11-15T00:46:04,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/1b5ac90667124c698e0a19fb15c2a6b3 is 50, key is test_row_0/B:col10/1731631564189/Put/seqid=0 2024-11-15T00:46:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:46:05,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742108_1284 (size=12301) 2024-11-15T00:46:05,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/1b5ac90667124c698e0a19fb15c2a6b3 2024-11-15T00:46:05,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T00:46:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:05,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:05,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/c2533afb60b3468aa18fa02fa5f0c7e3 is 50, key is test_row_0/C:col10/1731631564189/Put/seqid=0 2024-11-15T00:46:05,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742109_1285 (size=12301) 2024-11-15T00:46:05,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/c2533afb60b3468aa18fa02fa5f0c7e3 2024-11-15T00:46:05,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/88b712cdfcb24244a34ee50bccc861ea as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/88b712cdfcb24244a34ee50bccc861ea 2024-11-15T00:46:05,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/88b712cdfcb24244a34ee50bccc861ea, entries=150, sequenceid=535, filesize=12.0 K 2024-11-15T00:46:05,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/1b5ac90667124c698e0a19fb15c2a6b3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/1b5ac90667124c698e0a19fb15c2a6b3 2024-11-15T00:46:05,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/1b5ac90667124c698e0a19fb15c2a6b3, entries=150, sequenceid=535, filesize=12.0 K 2024-11-15T00:46:05,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/c2533afb60b3468aa18fa02fa5f0c7e3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/c2533afb60b3468aa18fa02fa5f0c7e3 2024-11-15T00:46:05,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/c2533afb60b3468aa18fa02fa5f0c7e3, entries=150, sequenceid=535, filesize=12.0 K 2024-11-15T00:46:05,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 12c7a62476f136ecef48b9c405ac1dfb in 636ms, sequenceid=535, compaction requested=true 2024-11-15T00:46:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:05,165 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:05,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:05,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:05,166 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:05,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:05,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:05,167 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:05,167 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:05,167 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/A is initiating minor compaction (all files) 2024-11-15T00:46:05,167 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/B is initiating minor compaction (all files) 2024-11-15T00:46:05,167 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/B in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:05,167 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/A in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:05,167 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/f6cd2574f8c14d5eb2959df5404033e6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0b61a11022f744838be66da94634d1b4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ef0228df312d4e11975383a2a0212a16, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/1b5ac90667124c698e0a19fb15c2a6b3] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=48.8 K 2024-11-15T00:46:05,167 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/062c69a01fd847868be4a666f97d7020, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/8731251151b5480f99b4e0d5bca31b6c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/4ae8edaff7124373af28e815a26e9722, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/88b712cdfcb24244a34ee50bccc861ea] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=51.2 K 2024-11-15T00:46:05,168 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 062c69a01fd847868be4a666f97d7020, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731631562133 2024-11-15T00:46:05,169 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f6cd2574f8c14d5eb2959df5404033e6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731631562133 2024-11-15T00:46:05,171 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b61a11022f744838be66da94634d1b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731631563082 2024-11-15T00:46:05,171 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8731251151b5480f99b4e0d5bca31b6c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731631563074 2024-11-15T00:46:05,172 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0228df312d4e11975383a2a0212a16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731631563753 2024-11-15T00:46:05,172 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae8edaff7124373af28e815a26e9722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731631563753 2024-11-15T00:46:05,173 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b5ac90667124c698e0a19fb15c2a6b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731631564179 2024-11-15T00:46:05,173 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88b712cdfcb24244a34ee50bccc861ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731631564179 2024-11-15T00:46:05,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T00:46:05,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:05,206 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:46:05,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:46:05,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:05,235 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#B#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:05,236 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/be69c49cb5224c2ca488700cc7016c1c is 50, key is test_row_0/B:col10/1731631564189/Put/seqid=0 2024-11-15T00:46:05,250 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:05,250 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/cacd1fc1d276428aa1fbfb9bb40f80f5 is 50, key is test_row_0/A:col10/1731631564189/Put/seqid=0 2024-11-15T00:46:05,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/946badfe36864a33b1f5d6b6915919da is 50, key is test_row_0/A:col10/1731631564559/Put/seqid=0 2024-11-15T00:46:05,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631625287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631625290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631625291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742111_1287 (size=13187) 2024-11-15T00:46:05,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742110_1286 (size=13187) 2024-11-15T00:46:05,371 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/be69c49cb5224c2ca488700cc7016c1c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/be69c49cb5224c2ca488700cc7016c1c 2024-11-15T00:46:05,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742112_1288 (size=12301) 2024-11-15T00:46:05,375 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/946badfe36864a33b1f5d6b6915919da 2024-11-15T00:46:05,380 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/B of 12c7a62476f136ecef48b9c405ac1dfb into be69c49cb5224c2ca488700cc7016c1c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:05,380 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:05,380 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/B, priority=12, startTime=1731631565165; duration=0sec 2024-11-15T00:46:05,380 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:05,380 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:B 2024-11-15T00:46:05,380 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:05,385 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:05,385 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/C is initiating minor compaction (all files) 2024-11-15T00:46:05,386 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/C in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:05,386 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/feb4457d4ae24e8db7624aac64e424b1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/8c7de2a6e4e64c9ba08a794134953525, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1fbe4e16daa4434da2e5d988c387598f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/c2533afb60b3468aa18fa02fa5f0c7e3] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=48.8 K 2024-11-15T00:46:05,388 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting feb4457d4ae24e8db7624aac64e424b1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731631562133 2024-11-15T00:46:05,389 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c7de2a6e4e64c9ba08a794134953525, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731631563082 2024-11-15T00:46:05,389 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fbe4e16daa4434da2e5d988c387598f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731631563753 2024-11-15T00:46:05,390 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c2533afb60b3468aa18fa02fa5f0c7e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731631564179 2024-11-15T00:46:05,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/0cb8902190374fc3a1d202905120d4c2 is 50, key is test_row_0/B:col10/1731631564559/Put/seqid=0 2024-11-15T00:46:05,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631625410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631625410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631625410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,431 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:05,432 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/2961b806bb9745c3acd77142375a04eb is 50, key is test_row_0/C:col10/1731631564189/Put/seqid=0 2024-11-15T00:46:05,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742113_1289 (size=12301) 2024-11-15T00:46:05,469 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/0cb8902190374fc3a1d202905120d4c2 2024-11-15T00:46:05,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742114_1290 (size=13187) 2024-11-15T00:46:05,491 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/2961b806bb9745c3acd77142375a04eb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2961b806bb9745c3acd77142375a04eb 2024-11-15T00:46:05,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e146ec754b154e91bb9334d873b08f93 is 50, key is test_row_0/C:col10/1731631564559/Put/seqid=0 2024-11-15T00:46:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:46:05,530 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/C of 12c7a62476f136ecef48b9c405ac1dfb into 2961b806bb9745c3acd77142375a04eb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:05,530 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:05,530 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/C, priority=12, startTime=1731631565166; duration=0sec 2024-11-15T00:46:05,530 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:05,530 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:C 2024-11-15T00:46:05,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742115_1291 (size=12301) 2024-11-15T00:46:05,549 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e146ec754b154e91bb9334d873b08f93 2024-11-15T00:46:05,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/946badfe36864a33b1f5d6b6915919da as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/946badfe36864a33b1f5d6b6915919da 2024-11-15T00:46:05,567 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/946badfe36864a33b1f5d6b6915919da, entries=150, sequenceid=547, filesize=12.0 K 2024-11-15T00:46:05,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/0cb8902190374fc3a1d202905120d4c2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0cb8902190374fc3a1d202905120d4c2 2024-11-15T00:46:05,575 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0cb8902190374fc3a1d202905120d4c2, entries=150, sequenceid=547, filesize=12.0 K 2024-11-15T00:46:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e146ec754b154e91bb9334d873b08f93 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e146ec754b154e91bb9334d873b08f93 2024-11-15T00:46:05,588 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e146ec754b154e91bb9334d873b08f93, entries=150, sequenceid=547, filesize=12.0 K 2024-11-15T00:46:05,590 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 12c7a62476f136ecef48b9c405ac1dfb in 383ms, sequenceid=547, compaction requested=false 2024-11-15T00:46:05,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:05,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:05,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T00:46:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T00:46:05,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T00:46:05,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 713 msec 2024-11-15T00:46:05,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 729 msec 2024-11-15T00:46:05,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T00:46:05,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:05,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:05,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:05,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:05,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:05,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:05,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/11accc89ce8849c1b49d90d1e840b882 is 50, key is test_row_0/A:col10/1731631565285/Put/seqid=0 2024-11-15T00:46:05,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742116_1292 (size=14741) 2024-11-15T00:46:05,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631625682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/11accc89ce8849c1b49d90d1e840b882 2024-11-15T00:46:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631625689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631625695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/dc8a4d5637b8494e8c030933e16c79dc is 50, key is test_row_0/B:col10/1731631565285/Put/seqid=0 2024-11-15T00:46:05,744 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/cacd1fc1d276428aa1fbfb9bb40f80f5 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cacd1fc1d276428aa1fbfb9bb40f80f5 2024-11-15T00:46:05,761 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/A of 12c7a62476f136ecef48b9c405ac1dfb into cacd1fc1d276428aa1fbfb9bb40f80f5(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:05,761 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:05,761 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/A, priority=12, startTime=1731631565165; duration=0sec 2024-11-15T00:46:05,761 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:05,761 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:A 2024-11-15T00:46:05,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742117_1293 (size=12301) 2024-11-15T00:46:05,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/dc8a4d5637b8494e8c030933e16c79dc 2024-11-15T00:46:05,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3614c7c233c94c3d8253ed5ed5f25a01 is 50, key is test_row_0/C:col10/1731631565285/Put/seqid=0 2024-11-15T00:46:05,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631625810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631625810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631625810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:05,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742118_1294 (size=12301) 2024-11-15T00:46:05,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3614c7c233c94c3d8253ed5ed5f25a01 2024-11-15T00:46:05,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/11accc89ce8849c1b49d90d1e840b882 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/11accc89ce8849c1b49d90d1e840b882 2024-11-15T00:46:05,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/11accc89ce8849c1b49d90d1e840b882, entries=200, sequenceid=570, filesize=14.4 K 2024-11-15T00:46:05,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/dc8a4d5637b8494e8c030933e16c79dc as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/dc8a4d5637b8494e8c030933e16c79dc 2024-11-15T00:46:05,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/dc8a4d5637b8494e8c030933e16c79dc, entries=150, sequenceid=570, filesize=12.0 K 2024-11-15T00:46:05,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/3614c7c233c94c3d8253ed5ed5f25a01 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3614c7c233c94c3d8253ed5ed5f25a01 2024-11-15T00:46:05,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3614c7c233c94c3d8253ed5ed5f25a01, entries=150, sequenceid=570, filesize=12.0 K 2024-11-15T00:46:05,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 12c7a62476f136ecef48b9c405ac1dfb in 271ms, sequenceid=570, compaction requested=true 2024-11-15T00:46:05,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:05,909 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:05,910 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:05,910 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/A is initiating minor compaction (all files) 2024-11-15T00:46:05,910 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/A in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:05,910 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cacd1fc1d276428aa1fbfb9bb40f80f5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/946badfe36864a33b1f5d6b6915919da, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/11accc89ce8849c1b49d90d1e840b882] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=39.3 K 2024-11-15T00:46:05,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:05,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:05,911 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting cacd1fc1d276428aa1fbfb9bb40f80f5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731631564179 2024-11-15T00:46:05,911 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 946badfe36864a33b1f5d6b6915919da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1731631564543 2024-11-15T00:46:05,911 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11accc89ce8849c1b49d90d1e840b882, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731631565285 2024-11-15T00:46:05,913 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:05,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:05,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:05,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:05,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:05,932 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:05,933 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/B is initiating minor compaction (all files) 2024-11-15T00:46:05,936 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/B in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:05,936 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/be69c49cb5224c2ca488700cc7016c1c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0cb8902190374fc3a1d202905120d4c2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/dc8a4d5637b8494e8c030933e16c79dc] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=36.9 K 2024-11-15T00:46:05,943 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting be69c49cb5224c2ca488700cc7016c1c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731631564179 2024-11-15T00:46:05,945 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cb8902190374fc3a1d202905120d4c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1731631564543 2024-11-15T00:46:05,947 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting dc8a4d5637b8494e8c030933e16c79dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731631565285 2024-11-15T00:46:05,970 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#A#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:05,971 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/d3b0c78a5ac24f4094667038edbf142c is 50, key is test_row_0/A:col10/1731631565285/Put/seqid=0 2024-11-15T00:46:05,984 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:05,985 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/57b4753ff2104646ba1de68f5442794e is 50, key is test_row_0/B:col10/1731631565285/Put/seqid=0 2024-11-15T00:46:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:46:06,030 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:06,031 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-15T00:46:06,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:46:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:46:06,034 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:06,035 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:06,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:06,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742119_1295 (size=13289) 2024-11-15T00:46:06,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742120_1296 (size=13289) 2024-11-15T00:46:06,050 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/d3b0c78a5ac24f4094667038edbf142c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/d3b0c78a5ac24f4094667038edbf142c 2024-11-15T00:46:06,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/0a6e3ba6eacc4175bb4ce95a824e0002 is 50, key is test_row_0/A:col10/1731631565687/Put/seqid=0 2024-11-15T00:46:06,061 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/57b4753ff2104646ba1de68f5442794e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/57b4753ff2104646ba1de68f5442794e 2024-11-15T00:46:06,066 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/A of 12c7a62476f136ecef48b9c405ac1dfb into d3b0c78a5ac24f4094667038edbf142c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:06,066 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:06,066 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/A, priority=13, startTime=1731631565909; duration=0sec 2024-11-15T00:46:06,067 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:06,067 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:A 2024-11-15T00:46:06,067 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:06,069 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:06,069 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/C is initiating minor compaction (all files) 2024-11-15T00:46:06,069 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/C in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,069 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2961b806bb9745c3acd77142375a04eb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e146ec754b154e91bb9334d873b08f93, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3614c7c233c94c3d8253ed5ed5f25a01] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=36.9 K 2024-11-15T00:46:06,070 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2961b806bb9745c3acd77142375a04eb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731631564179 2024-11-15T00:46:06,071 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting e146ec754b154e91bb9334d873b08f93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1731631564543 2024-11-15T00:46:06,072 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3614c7c233c94c3d8253ed5ed5f25a01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731631565285 2024-11-15T00:46:06,072 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/B of 12c7a62476f136ecef48b9c405ac1dfb into 57b4753ff2104646ba1de68f5442794e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:06,073 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:06,073 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/B, priority=13, startTime=1731631565913; duration=0sec 2024-11-15T00:46:06,073 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:06,073 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:B 2024-11-15T00:46:06,121 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:06,121 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/9cb37156817f421eb5f7df1e5e2c05c5 is 50, key is test_row_0/C:col10/1731631565285/Put/seqid=0 2024-11-15T00:46:06,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742121_1297 (size=14741) 2024-11-15T00:46:06,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/0a6e3ba6eacc4175bb4ce95a824e0002 2024-11-15T00:46:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:46:06,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/6008ca38171442b19aa40701fa805807 is 50, key is test_row_0/B:col10/1731631565687/Put/seqid=0 2024-11-15T00:46:06,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631626157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631626157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631626165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:46:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:06,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742122_1298 (size=13289) 2024-11-15T00:46:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,201 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/9cb37156817f421eb5f7df1e5e2c05c5 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/9cb37156817f421eb5f7df1e5e2c05c5 2024-11-15T00:46:06,209 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/C of 12c7a62476f136ecef48b9c405ac1dfb into 9cb37156817f421eb5f7df1e5e2c05c5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:06,209 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:06,209 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/C, priority=13, startTime=1731631565913; duration=0sec 2024-11-15T00:46:06,209 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:06,209 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:C 2024-11-15T00:46:06,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742123_1299 (size=12301) 2024-11-15T00:46:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631626290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631626290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631626290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:46:06,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:06,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:46:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42740 deadline: 1731631626358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42768 deadline: 1731631626359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,365 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in 12c7a62476f136ecef48b9c405ac1dfb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,371 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in 12c7a62476f136ecef48b9c405ac1dfb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631626510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:46:06,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631626510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631626512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/6008ca38171442b19aa40701fa805807 2024-11-15T00:46:06,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e71ca7b49e444d25b6af6b65a7065949 is 50, key is test_row_0/C:col10/1731631565687/Put/seqid=0 2024-11-15T00:46:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:46:06,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:46:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742124_1300 (size=12301) 2024-11-15T00:46:06,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e71ca7b49e444d25b6af6b65a7065949 2024-11-15T00:46:06,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/0a6e3ba6eacc4175bb4ce95a824e0002 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0a6e3ba6eacc4175bb4ce95a824e0002 2024-11-15T00:46:06,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0a6e3ba6eacc4175bb4ce95a824e0002, entries=200, sequenceid=583, filesize=14.4 K 2024-11-15T00:46:06,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/6008ca38171442b19aa40701fa805807 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/6008ca38171442b19aa40701fa805807 2024-11-15T00:46:06,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/6008ca38171442b19aa40701fa805807, entries=150, sequenceid=583, filesize=12.0 K 2024-11-15T00:46:06,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/e71ca7b49e444d25b6af6b65a7065949 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e71ca7b49e444d25b6af6b65a7065949 2024-11-15T00:46:06,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e71ca7b49e444d25b6af6b65a7065949, entries=150, sequenceid=583, filesize=12.0 K 2024-11-15T00:46:06,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 12c7a62476f136ecef48b9c405ac1dfb in 724ms, sequenceid=583, compaction requested=false 2024-11-15T00:46:06,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:06,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T00:46:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:06,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:46:06,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:06,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/b0fdcf0d09854005806a7c99535031b3 is 50, key is test_row_0/A:col10/1731631566820/Put/seqid=0 2024-11-15T00:46:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631626855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742125_1301 (size=14741) 2024-11-15T00:46:06,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631626862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631626865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/b0fdcf0d09854005806a7c99535031b3 2024-11-15T00:46:06,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/19a003f150474501a6310412ad1d4392 is 50, key is test_row_0/B:col10/1731631566820/Put/seqid=0 2024-11-15T00:46:06,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742126_1302 (size=12301) 2024-11-15T00:46:06,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631626969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631626981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631626981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:06,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:46:06,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:06,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:06,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:07,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:46:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:07,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:46:07,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631627180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631627189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631627199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:46:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:07,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:07,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/19a003f150474501a6310412ad1d4392 2024-11-15T00:46:07,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/24a4432d736a4052ab4e75a265374c65 is 50, key is test_row_0/C:col10/1731631566820/Put/seqid=0 2024-11-15T00:46:07,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742127_1303 (size=12301) 2024-11-15T00:46:07,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/24a4432d736a4052ab4e75a265374c65 2024-11-15T00:46:07,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/b0fdcf0d09854005806a7c99535031b3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b0fdcf0d09854005806a7c99535031b3 2024-11-15T00:46:07,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b0fdcf0d09854005806a7c99535031b3, entries=200, sequenceid=607, filesize=14.4 K 2024-11-15T00:46:07,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/19a003f150474501a6310412ad1d4392 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/19a003f150474501a6310412ad1d4392 2024-11-15T00:46:07,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/19a003f150474501a6310412ad1d4392, entries=150, sequenceid=607, filesize=12.0 K 2024-11-15T00:46:07,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/24a4432d736a4052ab4e75a265374c65 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/24a4432d736a4052ab4e75a265374c65 2024-11-15T00:46:07,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/24a4432d736a4052ab4e75a265374c65, entries=150, sequenceid=607, filesize=12.0 K 2024-11-15T00:46:07,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 12c7a62476f136ecef48b9c405ac1dfb in 624ms, sequenceid=607, compaction requested=true 2024-11-15T00:46:07,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:07,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:07,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:07,448 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:07,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:07,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:07,448 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:07,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:07,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:07,448 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:07,449 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/B is initiating minor compaction (all files) 2024-11-15T00:46:07,449 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/B in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:07,449 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/57b4753ff2104646ba1de68f5442794e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/6008ca38171442b19aa40701fa805807, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/19a003f150474501a6310412ad1d4392] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=37.0 K 2024-11-15T00:46:07,449 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:07,449 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/A is initiating minor compaction (all files) 2024-11-15T00:46:07,449 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/A in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:07,449 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 57b4753ff2104646ba1de68f5442794e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731631565285 2024-11-15T00:46:07,449 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/d3b0c78a5ac24f4094667038edbf142c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0a6e3ba6eacc4175bb4ce95a824e0002, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b0fdcf0d09854005806a7c99535031b3] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=41.8 K 2024-11-15T00:46:07,449 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 6008ca38171442b19aa40701fa805807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=583, earliestPutTs=1731631565659 2024-11-15T00:46:07,450 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b0c78a5ac24f4094667038edbf142c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731631565285 2024-11-15T00:46:07,450 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a6e3ba6eacc4175bb4ce95a824e0002, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=583, earliestPutTs=1731631565659 2024-11-15T00:46:07,450 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a003f150474501a6310412ad1d4392, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731631566142 2024-11-15T00:46:07,451 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0fdcf0d09854005806a7c99535031b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731631566142 2024-11-15T00:46:07,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:46:07,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:07,466 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T00:46:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:07,495 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#A#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:07,495 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/38a087e37cc444839dd46c1c029c15bc is 50, key is test_row_0/A:col10/1731631566820/Put/seqid=0 2024-11-15T00:46:07,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. as already flushing 2024-11-15T00:46:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:07,507 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:07,507 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/c0b209b0e41042818445ef3881c58a9a is 50, key is test_row_0/B:col10/1731631566820/Put/seqid=0 2024-11-15T00:46:07,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/cb6b939e0fe543619051ddb85510ef63 is 50, key is test_row_0/A:col10/1731631566860/Put/seqid=0 2024-11-15T00:46:07,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631627552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631627555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631627559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742129_1305 (size=13391) 2024-11-15T00:46:07,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742128_1304 (size=13391) 2024-11-15T00:46:07,587 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/38a087e37cc444839dd46c1c029c15bc as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/38a087e37cc444839dd46c1c029c15bc 2024-11-15T00:46:07,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742130_1306 (size=9857) 2024-11-15T00:46:07,623 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/A of 12c7a62476f136ecef48b9c405ac1dfb into 38a087e37cc444839dd46c1c029c15bc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:07,623 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:07,623 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/A, priority=13, startTime=1731631567447; duration=0sec 2024-11-15T00:46:07,623 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:07,623 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:A 2024-11-15T00:46:07,624 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:07,631 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:07,631 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/C is initiating minor compaction (all files) 2024-11-15T00:46:07,632 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/C in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:07,632 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/9cb37156817f421eb5f7df1e5e2c05c5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e71ca7b49e444d25b6af6b65a7065949, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/24a4432d736a4052ab4e75a265374c65] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=37.0 K 2024-11-15T00:46:07,632 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cb37156817f421eb5f7df1e5e2c05c5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731631565285 2024-11-15T00:46:07,632 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting e71ca7b49e444d25b6af6b65a7065949, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=583, earliestPutTs=1731631565659 2024-11-15T00:46:07,633 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24a4432d736a4052ab4e75a265374c65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731631566142 2024-11-15T00:46:07,662 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:07,663 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/9a21f15d4b8d427b92f28706e8445c28 is 50, key is test_row_0/C:col10/1731631566820/Put/seqid=0 2024-11-15T00:46:07,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631627670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631627672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631627672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742131_1307 (size=13391) 2024-11-15T00:46:07,766 INFO [Thread-1010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1010. 2024-11-15T00:46:07,766 DEBUG [Thread-1010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:07,766 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:07,766 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:07,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:07,769 INFO [Thread-1006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1006. 2024-11-15T00:46:07,769 INFO [Thread-1008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1008. 2024-11-15T00:46:07,769 DEBUG [Thread-1006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:07,769 DEBUG [Thread-1008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:07,769 DEBUG [Thread-1006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:07,769 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:07,769 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:07,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:07,770 DEBUG [Thread-1006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:07,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:07,770 INFO [Thread-1004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1004. 2024-11-15T00:46:07,770 DEBUG [Thread-1004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:07,770 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:07,770 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:07,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:07,776 INFO [Thread-1002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1002. 2024-11-15T00:46:07,776 DEBUG [Thread-1002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:07,776 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:07,777 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:07,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:07,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631627879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631627879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631627879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:07,972 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/c0b209b0e41042818445ef3881c58a9a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/c0b209b0e41042818445ef3881c58a9a 2024-11-15T00:46:07,977 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/B of 12c7a62476f136ecef48b9c405ac1dfb into c0b209b0e41042818445ef3881c58a9a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:07,977 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:07,977 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/B, priority=13, startTime=1731631567448; duration=0sec 2024-11-15T00:46:07,977 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:07,977 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:B 2024-11-15T00:46:08,001 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/cb6b939e0fe543619051ddb85510ef63 2024-11-15T00:46:08,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/3b77365e9582487cb70d972cbc0ca318 is 50, key is test_row_0/B:col10/1731631566860/Put/seqid=0 2024-11-15T00:46:08,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742132_1308 (size=9857) 2024-11-15T00:46:08,015 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/3b77365e9582487cb70d972cbc0ca318 2024-11-15T00:46:08,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/fbea2b8ccda741dbaac67a4ef71f1c82 is 50, key is test_row_0/C:col10/1731631566860/Put/seqid=0 2024-11-15T00:46:08,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742133_1309 (size=9857) 2024-11-15T00:46:08,110 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/9a21f15d4b8d427b92f28706e8445c28 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/9a21f15d4b8d427b92f28706e8445c28 2024-11-15T00:46:08,116 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/C of 12c7a62476f136ecef48b9c405ac1dfb into 9a21f15d4b8d427b92f28706e8445c28(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:08,116 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:08,116 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/C, priority=13, startTime=1731631567448; duration=0sec 2024-11-15T00:46:08,116 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:08,116 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:C 2024-11-15T00:46:08,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:08,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42724 deadline: 1731631628190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42708 deadline: 1731631628190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:08,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42764 deadline: 1731631628190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:08,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:46:08,431 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/fbea2b8ccda741dbaac67a4ef71f1c82 2024-11-15T00:46:08,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/cb6b939e0fe543619051ddb85510ef63 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb6b939e0fe543619051ddb85510ef63 2024-11-15T00:46:08,444 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb6b939e0fe543619051ddb85510ef63, entries=100, sequenceid=618, filesize=9.6 K 2024-11-15T00:46:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/3b77365e9582487cb70d972cbc0ca318 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/3b77365e9582487cb70d972cbc0ca318 2024-11-15T00:46:08,451 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/3b77365e9582487cb70d972cbc0ca318, entries=100, sequenceid=618, filesize=9.6 K 2024-11-15T00:46:08,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/fbea2b8ccda741dbaac67a4ef71f1c82 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/fbea2b8ccda741dbaac67a4ef71f1c82 2024-11-15T00:46:08,456 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/fbea2b8ccda741dbaac67a4ef71f1c82, entries=100, sequenceid=618, filesize=9.6 K 2024-11-15T00:46:08,456 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=120.76 KB/123660 for 12c7a62476f136ecef48b9c405ac1dfb in 991ms, sequenceid=618, compaction requested=false 2024-11-15T00:46:08,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:08,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:08,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-15T00:46:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-15T00:46:08,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T00:46:08,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4220 sec 2024-11-15T00:46:08,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.4290 sec 2024-11-15T00:46:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:08,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-15T00:46:08,701 INFO [Thread-995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-995. 2024-11-15T00:46:08,701 INFO [Thread-997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-997. 2024-11-15T00:46:08,701 DEBUG [Thread-995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:08,701 DEBUG [Thread-995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:08,701 DEBUG [Thread-997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:08,701 DEBUG [Thread-997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:08,701 DEBUG [Thread-995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:08,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:08,701 DEBUG [Thread-997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:08,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:08,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:08,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:08,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:08,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:08,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:08,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:08,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/6dcf3c95b580408ab63a277387d20db8 is 50, key is test_row_0/A:col10/1731631568699/Put/seqid=0 2024-11-15T00:46:08,711 INFO [Thread-993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-993. 2024-11-15T00:46:08,711 DEBUG [Thread-993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:08,711 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:08,711 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:08,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:08,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742134_1310 (size=12301) 2024-11-15T00:46:09,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/6dcf3c95b580408ab63a277387d20db8 2024-11-15T00:46:09,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/646f112c66ea42e99186adaeefe137b8 is 50, key is test_row_0/B:col10/1731631568699/Put/seqid=0 2024-11-15T00:46:09,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742135_1311 (size=12301) 2024-11-15T00:46:09,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/646f112c66ea42e99186adaeefe137b8 2024-11-15T00:46:09,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/926e66236d314962b5ca631d0cfbd5cd is 50, key is test_row_0/C:col10/1731631568699/Put/seqid=0 2024-11-15T00:46:09,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742136_1312 (size=12301) 2024-11-15T00:46:09,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/926e66236d314962b5ca631d0cfbd5cd 2024-11-15T00:46:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/6dcf3c95b580408ab63a277387d20db8 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6dcf3c95b580408ab63a277387d20db8 2024-11-15T00:46:09,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6dcf3c95b580408ab63a277387d20db8, entries=150, sequenceid=644, filesize=12.0 K 2024-11-15T00:46:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/646f112c66ea42e99186adaeefe137b8 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/646f112c66ea42e99186adaeefe137b8 2024-11-15T00:46:09,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/646f112c66ea42e99186adaeefe137b8, entries=150, sequenceid=644, filesize=12.0 K 2024-11-15T00:46:09,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/926e66236d314962b5ca631d0cfbd5cd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/926e66236d314962b5ca631d0cfbd5cd 2024-11-15T00:46:09,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/926e66236d314962b5ca631d0cfbd5cd, entries=150, sequenceid=644, filesize=12.0 K 2024-11-15T00:46:09,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=6.71 KB/6870 for 12c7a62476f136ecef48b9c405ac1dfb in 1282ms, sequenceid=644, compaction requested=true 2024-11-15T00:46:09,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:09,983 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:09,983 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12c7a62476f136ecef48b9c405ac1dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:09,984 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:09,984 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/B is initiating minor compaction (all files) 2024-11-15T00:46:09,984 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/B in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:09,984 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/c0b209b0e41042818445ef3881c58a9a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/3b77365e9582487cb70d972cbc0ca318, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/646f112c66ea42e99186adaeefe137b8] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=34.7 K 2024-11-15T00:46:09,985 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:09,985 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/A is initiating minor compaction (all files) 2024-11-15T00:46:09,985 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/A in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:09,985 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/38a087e37cc444839dd46c1c029c15bc, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb6b939e0fe543619051ddb85510ef63, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6dcf3c95b580408ab63a277387d20db8] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=34.7 K 2024-11-15T00:46:09,985 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38a087e37cc444839dd46c1c029c15bc, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731631566142 2024-11-15T00:46:09,986 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c0b209b0e41042818445ef3881c58a9a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731631566142 2024-11-15T00:46:09,986 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb6b939e0fe543619051ddb85510ef63, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731631566860 2024-11-15T00:46:09,986 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b77365e9582487cb70d972cbc0ca318, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731631566860 2024-11-15T00:46:09,986 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dcf3c95b580408ab63a277387d20db8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=644, earliestPutTs=1731631567556 2024-11-15T00:46:09,987 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 646f112c66ea42e99186adaeefe137b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=644, earliestPutTs=1731631567556 2024-11-15T00:46:10,011 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#B#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:10,012 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/a9f32d3dbb884d0588dea7f6c7ca9d09 is 50, key is test_row_0/B:col10/1731631568699/Put/seqid=0 2024-11-15T00:46:10,014 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#A#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:10,015 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/5ed0f05f28bd46d6833d5f7fa4f495fe is 50, key is test_row_0/A:col10/1731631568699/Put/seqid=0 2024-11-15T00:46:10,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742137_1313 (size=13493) 2024-11-15T00:46:10,028 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/a9f32d3dbb884d0588dea7f6c7ca9d09 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a9f32d3dbb884d0588dea7f6c7ca9d09 2024-11-15T00:46:10,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742138_1314 (size=13493) 2024-11-15T00:46:10,036 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/B of 12c7a62476f136ecef48b9c405ac1dfb into a9f32d3dbb884d0588dea7f6c7ca9d09(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:10,036 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:10,036 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/B, priority=13, startTime=1731631569983; duration=0sec 2024-11-15T00:46:10,036 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:10,036 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:B 2024-11-15T00:46:10,037 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:10,040 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/5ed0f05f28bd46d6833d5f7fa4f495fe as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/5ed0f05f28bd46d6833d5f7fa4f495fe 2024-11-15T00:46:10,040 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:10,040 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 12c7a62476f136ecef48b9c405ac1dfb/C is initiating minor compaction (all files) 2024-11-15T00:46:10,040 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12c7a62476f136ecef48b9c405ac1dfb/C in TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:10,040 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/9a21f15d4b8d427b92f28706e8445c28, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/fbea2b8ccda741dbaac67a4ef71f1c82, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/926e66236d314962b5ca631d0cfbd5cd] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp, totalSize=34.7 K 2024-11-15T00:46:10,041 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a21f15d4b8d427b92f28706e8445c28, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731631566142 2024-11-15T00:46:10,042 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting fbea2b8ccda741dbaac67a4ef71f1c82, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731631566860 2024-11-15T00:46:10,044 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 926e66236d314962b5ca631d0cfbd5cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=644, earliestPutTs=1731631567556 2024-11-15T00:46:10,058 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/A of 12c7a62476f136ecef48b9c405ac1dfb into 5ed0f05f28bd46d6833d5f7fa4f495fe(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:10,058 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:10,058 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/A, priority=13, startTime=1731631569983; duration=0sec 2024-11-15T00:46:10,058 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:10,058 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:A 2024-11-15T00:46:10,065 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12c7a62476f136ecef48b9c405ac1dfb#C#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:10,065 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/44d073f7936d44d187ac95961ac1ff10 is 50, key is test_row_0/C:col10/1731631568699/Put/seqid=0 2024-11-15T00:46:10,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742139_1315 (size=13493) 2024-11-15T00:46:10,102 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/44d073f7936d44d187ac95961ac1ff10 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/44d073f7936d44d187ac95961ac1ff10 2024-11-15T00:46:10,107 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12c7a62476f136ecef48b9c405ac1dfb/C of 12c7a62476f136ecef48b9c405ac1dfb into 44d073f7936d44d187ac95961ac1ff10(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:10,107 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12c7a62476f136ecef48b9c405ac1dfb: 2024-11-15T00:46:10,107 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., storeName=12c7a62476f136ecef48b9c405ac1dfb/C, priority=13, startTime=1731631569983; duration=0sec 2024-11-15T00:46:10,107 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:10,107 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12c7a62476f136ecef48b9c405ac1dfb:C 2024-11-15T00:46:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:46:10,213 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:10,392 INFO [Thread-999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-999. 2024-11-15T00:46:10,392 DEBUG [Thread-999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:10,392 DEBUG [Thread-999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:10,392 DEBUG [Thread-999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:10,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:10,401 INFO [Thread-991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-991. 2024-11-15T00:46:10,401 DEBUG [Thread-991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:10,401 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:10,401 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:10,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:10,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:46:10,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-15T00:46:10,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-15T00:46:10,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-15T00:46:10,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-11-15T00:46:10,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-15T00:46:10,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:46:10,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2113 2024-11-15T00:46:10,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1975 2024-11-15T00:46:10,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2016 2024-11-15T00:46:10,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2095 2024-11-15T00:46:10,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1976 2024-11-15T00:46:10,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:46:10,402 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:46:10,402 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:46:10,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:10,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:10,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:10,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:46:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:46:10,410 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631570409"}]},"ts":"1731631570409"} 2024-11-15T00:46:10,412 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:46:10,412 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:46:10,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:46:10,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12c7a62476f136ecef48b9c405ac1dfb, UNASSIGN}] 2024-11-15T00:46:10,417 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12c7a62476f136ecef48b9c405ac1dfb, UNASSIGN 2024-11-15T00:46:10,418 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=12c7a62476f136ecef48b9c405ac1dfb, regionState=CLOSING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:10,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12c7a62476f136ecef48b9c405ac1dfb, UNASSIGN because future has completed 2024-11-15T00:46:10,420 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:46:10,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:46:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:46:10,573 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing 12c7a62476f136ecef48b9c405ac1dfb, disabling compactions & flushes 2024-11-15T00:46:10,574 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. after waiting 0 ms 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:10,574 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing 12c7a62476f136ecef48b9c405ac1dfb 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=A 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=B 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12c7a62476f136ecef48b9c405ac1dfb, store=C 2024-11-15T00:46:10,574 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:10,579 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2028bc5b85424b57bf83f4b0f2fcc549 is 50, key is test_row_0/A:col10/1731631570390/Put/seqid=0 2024-11-15T00:46:10,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742140_1316 (size=12301) 2024-11-15T00:46:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:46:10,986 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2028bc5b85424b57bf83f4b0f2fcc549 2024-11-15T00:46:10,993 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/dbf07671495443e0b863cea39cc5e0cd is 50, key is test_row_0/B:col10/1731631570390/Put/seqid=0 2024-11-15T00:46:10,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742141_1317 (size=12301) 2024-11-15T00:46:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:46:11,397 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/dbf07671495443e0b863cea39cc5e0cd 2024-11-15T00:46:11,403 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/b195029dbbdf435bbb74667e8684d3cb is 50, key is test_row_0/C:col10/1731631570390/Put/seqid=0 2024-11-15T00:46:11,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742142_1318 (size=12301) 2024-11-15T00:46:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:46:11,807 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/b195029dbbdf435bbb74667e8684d3cb 2024-11-15T00:46:11,813 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/A/2028bc5b85424b57bf83f4b0f2fcc549 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2028bc5b85424b57bf83f4b0f2fcc549 2024-11-15T00:46:11,819 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2028bc5b85424b57bf83f4b0f2fcc549, entries=150, sequenceid=653, filesize=12.0 K 2024-11-15T00:46:11,821 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/B/dbf07671495443e0b863cea39cc5e0cd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/dbf07671495443e0b863cea39cc5e0cd 2024-11-15T00:46:11,825 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/dbf07671495443e0b863cea39cc5e0cd, entries=150, sequenceid=653, filesize=12.0 K 2024-11-15T00:46:11,825 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/.tmp/C/b195029dbbdf435bbb74667e8684d3cb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/b195029dbbdf435bbb74667e8684d3cb 2024-11-15T00:46:11,829 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/b195029dbbdf435bbb74667e8684d3cb, entries=150, sequenceid=653, filesize=12.0 K 2024-11-15T00:46:11,829 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 12c7a62476f136ecef48b9c405ac1dfb in 1255ms, sequenceid=653, compaction requested=false 2024-11-15T00:46:11,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/72cf5c1a006c442cb852c251298c386d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/e5a8b31636d74f9eac98dc56f7555b22, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/20c3fd06923d4e98b79b032335c6d202, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/bbcf6d1bad9e451eb5d8b9625d4e5d08, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7adc91fdeaa846c0a8670bbea5fb6a46, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0aa35c1048574ac9bfa6e430d87bc1f1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2b134e9b72764bfaad478968b8fd7e67, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb7a724fa83946fd81efdee54f772902, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/a5bd125137f748c8943cf0b8267b7c11, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6da8156b87154d11a52a117b9f86e7ff, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/eea733d943f6404ba8b4b380f8cd96d6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2ada9c6f168243ccb98f6eb327e36722, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f5599cb560ac4713bbe301e5964dd86f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/1cec7252bef7449ba53b40c89b186299, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2a0024af6bce4c58ad5c73d972b49263, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/9ff7c608a6ae420793637b8ae1dab096, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7e830d5ab02845a98e6b9a2a568123ca, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/acec81ae3ae44794be87d82e973e4159, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/ceaff70dd87140559ecaaac990c19e50, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/131e077b8679450c90da66c477442c6e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b3256587af43472eb0e7904f5be63ead, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/062c69a01fd847868be4a666f97d7020, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f2cd4773eb034a5f8d83da8c7340a348, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/8731251151b5480f99b4e0d5bca31b6c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/4ae8edaff7124373af28e815a26e9722, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cacd1fc1d276428aa1fbfb9bb40f80f5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/88b712cdfcb24244a34ee50bccc861ea, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/946badfe36864a33b1f5d6b6915919da, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/11accc89ce8849c1b49d90d1e840b882, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/d3b0c78a5ac24f4094667038edbf142c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0a6e3ba6eacc4175bb4ce95a824e0002, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b0fdcf0d09854005806a7c99535031b3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/38a087e37cc444839dd46c1c029c15bc, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb6b939e0fe543619051ddb85510ef63, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6dcf3c95b580408ab63a277387d20db8] to archive 2024-11-15T00:46:11,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:46:11,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/72cf5c1a006c442cb852c251298c386d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/72cf5c1a006c442cb852c251298c386d 2024-11-15T00:46:11,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/e5a8b31636d74f9eac98dc56f7555b22 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/e5a8b31636d74f9eac98dc56f7555b22 2024-11-15T00:46:11,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/20c3fd06923d4e98b79b032335c6d202 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/20c3fd06923d4e98b79b032335c6d202 2024-11-15T00:46:11,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/bbcf6d1bad9e451eb5d8b9625d4e5d08 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/bbcf6d1bad9e451eb5d8b9625d4e5d08 2024-11-15T00:46:11,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7adc91fdeaa846c0a8670bbea5fb6a46 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7adc91fdeaa846c0a8670bbea5fb6a46 2024-11-15T00:46:11,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0aa35c1048574ac9bfa6e430d87bc1f1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0aa35c1048574ac9bfa6e430d87bc1f1 2024-11-15T00:46:11,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2b134e9b72764bfaad478968b8fd7e67 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2b134e9b72764bfaad478968b8fd7e67 2024-11-15T00:46:11,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb7a724fa83946fd81efdee54f772902 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb7a724fa83946fd81efdee54f772902 2024-11-15T00:46:11,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/a5bd125137f748c8943cf0b8267b7c11 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/a5bd125137f748c8943cf0b8267b7c11 2024-11-15T00:46:11,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6da8156b87154d11a52a117b9f86e7ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6da8156b87154d11a52a117b9f86e7ff 2024-11-15T00:46:11,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/eea733d943f6404ba8b4b380f8cd96d6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/eea733d943f6404ba8b4b380f8cd96d6 2024-11-15T00:46:11,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2ada9c6f168243ccb98f6eb327e36722 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2ada9c6f168243ccb98f6eb327e36722 2024-11-15T00:46:11,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f5599cb560ac4713bbe301e5964dd86f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f5599cb560ac4713bbe301e5964dd86f 2024-11-15T00:46:11,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/1cec7252bef7449ba53b40c89b186299 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/1cec7252bef7449ba53b40c89b186299 2024-11-15T00:46:11,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2a0024af6bce4c58ad5c73d972b49263 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2a0024af6bce4c58ad5c73d972b49263 2024-11-15T00:46:11,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/9ff7c608a6ae420793637b8ae1dab096 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/9ff7c608a6ae420793637b8ae1dab096 2024-11-15T00:46:11,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7e830d5ab02845a98e6b9a2a568123ca to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/7e830d5ab02845a98e6b9a2a568123ca 2024-11-15T00:46:11,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/acec81ae3ae44794be87d82e973e4159 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/acec81ae3ae44794be87d82e973e4159 2024-11-15T00:46:11,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/ceaff70dd87140559ecaaac990c19e50 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/ceaff70dd87140559ecaaac990c19e50 2024-11-15T00:46:11,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/131e077b8679450c90da66c477442c6e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/131e077b8679450c90da66c477442c6e 2024-11-15T00:46:11,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b3256587af43472eb0e7904f5be63ead to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b3256587af43472eb0e7904f5be63ead 2024-11-15T00:46:11,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/062c69a01fd847868be4a666f97d7020 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/062c69a01fd847868be4a666f97d7020 2024-11-15T00:46:11,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f2cd4773eb034a5f8d83da8c7340a348 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/f2cd4773eb034a5f8d83da8c7340a348 2024-11-15T00:46:11,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/8731251151b5480f99b4e0d5bca31b6c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/8731251151b5480f99b4e0d5bca31b6c 2024-11-15T00:46:11,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/4ae8edaff7124373af28e815a26e9722 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/4ae8edaff7124373af28e815a26e9722 2024-11-15T00:46:11,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cacd1fc1d276428aa1fbfb9bb40f80f5 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cacd1fc1d276428aa1fbfb9bb40f80f5 2024-11-15T00:46:11,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/88b712cdfcb24244a34ee50bccc861ea to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/88b712cdfcb24244a34ee50bccc861ea 2024-11-15T00:46:11,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/946badfe36864a33b1f5d6b6915919da to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/946badfe36864a33b1f5d6b6915919da 2024-11-15T00:46:11,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/11accc89ce8849c1b49d90d1e840b882 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/11accc89ce8849c1b49d90d1e840b882 2024-11-15T00:46:11,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/d3b0c78a5ac24f4094667038edbf142c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/d3b0c78a5ac24f4094667038edbf142c 2024-11-15T00:46:11,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0a6e3ba6eacc4175bb4ce95a824e0002 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/0a6e3ba6eacc4175bb4ce95a824e0002 2024-11-15T00:46:11,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b0fdcf0d09854005806a7c99535031b3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/b0fdcf0d09854005806a7c99535031b3 2024-11-15T00:46:11,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/38a087e37cc444839dd46c1c029c15bc to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/38a087e37cc444839dd46c1c029c15bc 2024-11-15T00:46:11,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb6b939e0fe543619051ddb85510ef63 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/cb6b939e0fe543619051ddb85510ef63 2024-11-15T00:46:11,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6dcf3c95b580408ab63a277387d20db8 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/6dcf3c95b580408ab63a277387d20db8 2024-11-15T00:46:11,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/8f1ffe02a5984bbc8abecc16902081d9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/048fd3ee4ca2423cb39b32f9947c9325, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/417581b7db084db1877e41a8acbecfe6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ec4c4b13cfdc4df2b235fb3e539737e6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/87a0acc84444459e9600a9ae7467b2c9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/32fc602af1d646ee81ecb23b0990ea48, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2793f0db4d2d4913bc663e8a6eaa34ed, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e5b674a608304c119c35693b93f11505, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a4e1ade85d994d248b9ae43ce9872c0a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/4ac827467de340c0b79810e595c51959, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7ddd59204d14634a731c84c4bf3f1ba, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/eeec8bc65c464694acd078ac3c9682fd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/5cb7c7d1ebfc446c9ac337ed047a4aeb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e8575b7ffc5f48ecb03c12ba2ad1351b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/b7eab77deda349c6b63e936f29467359, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e173f937c25d4910b4b51e3ffeac84cc, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a9b61758ad524df29e7b7887146e4449, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/596bfbf92c144108ade32af160667eca, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7c2d51074074e3fbe2a9e7d77b15a59, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/11111651c2b74e17a09c648ad3bf0297, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2b5e9a1cc2b148c1b30efcaf50a4e1c2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/f6cd2574f8c14d5eb2959df5404033e6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/89ca7d71418c42f78de6d5a9533ae51e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0b61a11022f744838be66da94634d1b4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ef0228df312d4e11975383a2a0212a16, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/be69c49cb5224c2ca488700cc7016c1c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/1b5ac90667124c698e0a19fb15c2a6b3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0cb8902190374fc3a1d202905120d4c2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/57b4753ff2104646ba1de68f5442794e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/dc8a4d5637b8494e8c030933e16c79dc, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/6008ca38171442b19aa40701fa805807, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/c0b209b0e41042818445ef3881c58a9a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/19a003f150474501a6310412ad1d4392, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/3b77365e9582487cb70d972cbc0ca318, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/646f112c66ea42e99186adaeefe137b8] to archive 2024-11-15T00:46:11,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:46:11,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/8f1ffe02a5984bbc8abecc16902081d9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/8f1ffe02a5984bbc8abecc16902081d9 2024-11-15T00:46:11,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/048fd3ee4ca2423cb39b32f9947c9325 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/048fd3ee4ca2423cb39b32f9947c9325 2024-11-15T00:46:11,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/417581b7db084db1877e41a8acbecfe6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/417581b7db084db1877e41a8acbecfe6 2024-11-15T00:46:11,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ec4c4b13cfdc4df2b235fb3e539737e6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ec4c4b13cfdc4df2b235fb3e539737e6 2024-11-15T00:46:11,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/87a0acc84444459e9600a9ae7467b2c9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/87a0acc84444459e9600a9ae7467b2c9 2024-11-15T00:46:11,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/32fc602af1d646ee81ecb23b0990ea48 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/32fc602af1d646ee81ecb23b0990ea48 2024-11-15T00:46:11,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2793f0db4d2d4913bc663e8a6eaa34ed to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2793f0db4d2d4913bc663e8a6eaa34ed 2024-11-15T00:46:11,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e5b674a608304c119c35693b93f11505 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e5b674a608304c119c35693b93f11505 2024-11-15T00:46:11,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a4e1ade85d994d248b9ae43ce9872c0a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a4e1ade85d994d248b9ae43ce9872c0a 2024-11-15T00:46:11,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/4ac827467de340c0b79810e595c51959 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/4ac827467de340c0b79810e595c51959 2024-11-15T00:46:11,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7ddd59204d14634a731c84c4bf3f1ba to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7ddd59204d14634a731c84c4bf3f1ba 2024-11-15T00:46:11,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/eeec8bc65c464694acd078ac3c9682fd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/eeec8bc65c464694acd078ac3c9682fd 2024-11-15T00:46:11,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/5cb7c7d1ebfc446c9ac337ed047a4aeb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/5cb7c7d1ebfc446c9ac337ed047a4aeb 2024-11-15T00:46:11,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e8575b7ffc5f48ecb03c12ba2ad1351b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e8575b7ffc5f48ecb03c12ba2ad1351b 2024-11-15T00:46:11,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/b7eab77deda349c6b63e936f29467359 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/b7eab77deda349c6b63e936f29467359 2024-11-15T00:46:11,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e173f937c25d4910b4b51e3ffeac84cc to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e173f937c25d4910b4b51e3ffeac84cc 2024-11-15T00:46:11,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a9b61758ad524df29e7b7887146e4449 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a9b61758ad524df29e7b7887146e4449 2024-11-15T00:46:11,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/596bfbf92c144108ade32af160667eca to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/596bfbf92c144108ade32af160667eca 2024-11-15T00:46:11,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7c2d51074074e3fbe2a9e7d77b15a59 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/e7c2d51074074e3fbe2a9e7d77b15a59 2024-11-15T00:46:11,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/11111651c2b74e17a09c648ad3bf0297 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/11111651c2b74e17a09c648ad3bf0297 2024-11-15T00:46:11,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2b5e9a1cc2b148c1b30efcaf50a4e1c2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/2b5e9a1cc2b148c1b30efcaf50a4e1c2 2024-11-15T00:46:11,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/f6cd2574f8c14d5eb2959df5404033e6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/f6cd2574f8c14d5eb2959df5404033e6 2024-11-15T00:46:11,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/89ca7d71418c42f78de6d5a9533ae51e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/89ca7d71418c42f78de6d5a9533ae51e 2024-11-15T00:46:11,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0b61a11022f744838be66da94634d1b4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0b61a11022f744838be66da94634d1b4 2024-11-15T00:46:11,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ef0228df312d4e11975383a2a0212a16 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/ef0228df312d4e11975383a2a0212a16 2024-11-15T00:46:11,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/be69c49cb5224c2ca488700cc7016c1c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/be69c49cb5224c2ca488700cc7016c1c 2024-11-15T00:46:11,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/1b5ac90667124c698e0a19fb15c2a6b3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/1b5ac90667124c698e0a19fb15c2a6b3 2024-11-15T00:46:11,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0cb8902190374fc3a1d202905120d4c2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/0cb8902190374fc3a1d202905120d4c2 2024-11-15T00:46:11,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/57b4753ff2104646ba1de68f5442794e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/57b4753ff2104646ba1de68f5442794e 2024-11-15T00:46:11,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/dc8a4d5637b8494e8c030933e16c79dc to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/dc8a4d5637b8494e8c030933e16c79dc 2024-11-15T00:46:11,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/6008ca38171442b19aa40701fa805807 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/6008ca38171442b19aa40701fa805807 2024-11-15T00:46:11,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/c0b209b0e41042818445ef3881c58a9a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/c0b209b0e41042818445ef3881c58a9a 2024-11-15T00:46:11,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/19a003f150474501a6310412ad1d4392 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/19a003f150474501a6310412ad1d4392 2024-11-15T00:46:11,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/3b77365e9582487cb70d972cbc0ca318 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/3b77365e9582487cb70d972cbc0ca318 2024-11-15T00:46:11,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/646f112c66ea42e99186adaeefe137b8 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/646f112c66ea42e99186adaeefe137b8 2024-11-15T00:46:11,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/72094f11f9934b9d871eee6fe9292108, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/840becc3f65a4a5ab2ad271529748b6c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3c1c96e07e8a4fa08f943b35e93914a6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/66ba26dd15954abfb1a629acff321933, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/a34ea9f6db9d4798b5248ac65f212751, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f1349f6bfd184a74b20048cf0c5bcd32, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/0627126cc6784d8bb9a073ec1839d2a5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/836941d8fa1547279f03ee39da822a66, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/6f7304baee444448b277969808e2fdd6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/00de52224554481983f35de2628eff9f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/b53363151bc940e7ac2df61401455cd6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/88eb65b8f4604091be49c9d454766911, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/aab220d33f6049acb1c5c5957d00657e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2b67267a700e47e5952276182811bc01, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/28463d14233943e49f4c40cb2178fe7b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3e278c9320f34c2ea1ebd9081d4e7878, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1e6b7ebfca024e57a56dfdc9ef9c0627, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e160a3a7b20445c3a79d16fb3e0b746a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e0f69023ea4049ceb720b1bb6589de40, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3d3d94f5a0c541baa3336f3d3a02be37, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f07469c97f7b4958a3a41c73f9d741e6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/feb4457d4ae24e8db7624aac64e424b1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/46fb192e44cf49d4b6d593278c74130a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/8c7de2a6e4e64c9ba08a794134953525, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1fbe4e16daa4434da2e5d988c387598f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2961b806bb9745c3acd77142375a04eb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/c2533afb60b3468aa18fa02fa5f0c7e3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e146ec754b154e91bb9334d873b08f93, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/9cb37156817f421eb5f7df1e5e2c05c5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3614c7c233c94c3d8253ed5ed5f25a01, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e71ca7b49e444d25b6af6b65a7065949, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/9a21f15d4b8d427b92f28706e8445c28, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/24a4432d736a4052ab4e75a265374c65, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/fbea2b8ccda741dbaac67a4ef71f1c82, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/926e66236d314962b5ca631d0cfbd5cd] to archive 2024-11-15T00:46:11,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:46:11,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/72094f11f9934b9d871eee6fe9292108 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/72094f11f9934b9d871eee6fe9292108 2024-11-15T00:46:11,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/840becc3f65a4a5ab2ad271529748b6c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/840becc3f65a4a5ab2ad271529748b6c 2024-11-15T00:46:11,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3c1c96e07e8a4fa08f943b35e93914a6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3c1c96e07e8a4fa08f943b35e93914a6 2024-11-15T00:46:11,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/66ba26dd15954abfb1a629acff321933 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/66ba26dd15954abfb1a629acff321933 2024-11-15T00:46:11,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/a34ea9f6db9d4798b5248ac65f212751 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/a34ea9f6db9d4798b5248ac65f212751 2024-11-15T00:46:11,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f1349f6bfd184a74b20048cf0c5bcd32 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f1349f6bfd184a74b20048cf0c5bcd32 2024-11-15T00:46:11,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/0627126cc6784d8bb9a073ec1839d2a5 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/0627126cc6784d8bb9a073ec1839d2a5 2024-11-15T00:46:11,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/836941d8fa1547279f03ee39da822a66 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/836941d8fa1547279f03ee39da822a66 2024-11-15T00:46:11,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/6f7304baee444448b277969808e2fdd6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/6f7304baee444448b277969808e2fdd6 2024-11-15T00:46:11,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/00de52224554481983f35de2628eff9f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/00de52224554481983f35de2628eff9f 2024-11-15T00:46:11,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/b53363151bc940e7ac2df61401455cd6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/b53363151bc940e7ac2df61401455cd6 2024-11-15T00:46:11,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/88eb65b8f4604091be49c9d454766911 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/88eb65b8f4604091be49c9d454766911 2024-11-15T00:46:11,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/aab220d33f6049acb1c5c5957d00657e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/aab220d33f6049acb1c5c5957d00657e 2024-11-15T00:46:11,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2b67267a700e47e5952276182811bc01 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2b67267a700e47e5952276182811bc01 2024-11-15T00:46:11,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/28463d14233943e49f4c40cb2178fe7b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/28463d14233943e49f4c40cb2178fe7b 2024-11-15T00:46:11,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3e278c9320f34c2ea1ebd9081d4e7878 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3e278c9320f34c2ea1ebd9081d4e7878 2024-11-15T00:46:11,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1e6b7ebfca024e57a56dfdc9ef9c0627 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1e6b7ebfca024e57a56dfdc9ef9c0627 2024-11-15T00:46:11,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e160a3a7b20445c3a79d16fb3e0b746a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e160a3a7b20445c3a79d16fb3e0b746a 2024-11-15T00:46:11,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e0f69023ea4049ceb720b1bb6589de40 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e0f69023ea4049ceb720b1bb6589de40 2024-11-15T00:46:11,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3d3d94f5a0c541baa3336f3d3a02be37 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3d3d94f5a0c541baa3336f3d3a02be37 2024-11-15T00:46:11,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f07469c97f7b4958a3a41c73f9d741e6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/f07469c97f7b4958a3a41c73f9d741e6 2024-11-15T00:46:11,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/feb4457d4ae24e8db7624aac64e424b1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/feb4457d4ae24e8db7624aac64e424b1 2024-11-15T00:46:11,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/46fb192e44cf49d4b6d593278c74130a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/46fb192e44cf49d4b6d593278c74130a 2024-11-15T00:46:11,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/8c7de2a6e4e64c9ba08a794134953525 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/8c7de2a6e4e64c9ba08a794134953525 2024-11-15T00:46:11,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1fbe4e16daa4434da2e5d988c387598f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/1fbe4e16daa4434da2e5d988c387598f 2024-11-15T00:46:11,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2961b806bb9745c3acd77142375a04eb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/2961b806bb9745c3acd77142375a04eb 2024-11-15T00:46:11,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/c2533afb60b3468aa18fa02fa5f0c7e3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/c2533afb60b3468aa18fa02fa5f0c7e3 2024-11-15T00:46:11,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e146ec754b154e91bb9334d873b08f93 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e146ec754b154e91bb9334d873b08f93 2024-11-15T00:46:11,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/9cb37156817f421eb5f7df1e5e2c05c5 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/9cb37156817f421eb5f7df1e5e2c05c5 2024-11-15T00:46:11,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3614c7c233c94c3d8253ed5ed5f25a01 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/3614c7c233c94c3d8253ed5ed5f25a01 2024-11-15T00:46:11,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e71ca7b49e444d25b6af6b65a7065949 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/e71ca7b49e444d25b6af6b65a7065949 2024-11-15T00:46:11,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/9a21f15d4b8d427b92f28706e8445c28 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/9a21f15d4b8d427b92f28706e8445c28 2024-11-15T00:46:11,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/24a4432d736a4052ab4e75a265374c65 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/24a4432d736a4052ab4e75a265374c65 2024-11-15T00:46:11,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/fbea2b8ccda741dbaac67a4ef71f1c82 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/fbea2b8ccda741dbaac67a4ef71f1c82 2024-11-15T00:46:11,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/926e66236d314962b5ca631d0cfbd5cd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/926e66236d314962b5ca631d0cfbd5cd 2024-11-15T00:46:11,936 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/recovered.edits/656.seqid, newMaxSeqId=656, maxSeqId=1 2024-11-15T00:46:11,936 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. 2024-11-15T00:46:11,936 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for 12c7a62476f136ecef48b9c405ac1dfb: Waiting for close lock at 1731631570574Running coprocessor pre-close hooks at 1731631570574Disabling compacts and flushes for region at 1731631570574Disabling writes for close at 1731631570574Obtaining lock to block concurrent updates at 1731631570574Preparing flush snapshotting stores in 12c7a62476f136ecef48b9c405ac1dfb at 1731631570574Finished memstore snapshotting TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731631570575 (+1 ms)Flushing stores of TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb. at 1731631570575Flushing 12c7a62476f136ecef48b9c405ac1dfb/A: creating writer at 1731631570575Flushing 12c7a62476f136ecef48b9c405ac1dfb/A: appending metadata at 1731631570578 (+3 ms)Flushing 12c7a62476f136ecef48b9c405ac1dfb/A: closing flushed file at 1731631570578Flushing 12c7a62476f136ecef48b9c405ac1dfb/B: creating writer at 1731631570990 (+412 ms)Flushing 12c7a62476f136ecef48b9c405ac1dfb/B: appending metadata at 1731631570992 (+2 ms)Flushing 12c7a62476f136ecef48b9c405ac1dfb/B: closing flushed file at 1731631570992Flushing 12c7a62476f136ecef48b9c405ac1dfb/C: creating writer at 1731631571401 (+409 ms)Flushing 12c7a62476f136ecef48b9c405ac1dfb/C: appending metadata at 1731631571403 (+2 ms)Flushing 12c7a62476f136ecef48b9c405ac1dfb/C: closing flushed file at 1731631571403Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e8cd102: reopening flushed file at 1731631571812 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51eefb30: reopening flushed file at 1731631571819 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25d59261: reopening flushed file at 1731631571825 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 12c7a62476f136ecef48b9c405ac1dfb in 1255ms, sequenceid=653, compaction requested=false at 1731631571829 (+4 ms)Writing region close event to WAL at 1731631571933 (+104 ms)Running coprocessor post-close hooks at 1731631571936 (+3 ms)Closed at 1731631571936 2024-11-15T00:46:11,938 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed 12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:11,938 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=12c7a62476f136ecef48b9c405ac1dfb, regionState=CLOSED 2024-11-15T00:46:11,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:46:11,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-15T00:46:11,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure 12c7a62476f136ecef48b9c405ac1dfb, server=fb2f6cb7846a,34537,1731631472208 in 1.5200 sec 2024-11-15T00:46:11,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-15T00:46:11,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12c7a62476f136ecef48b9c405ac1dfb, UNASSIGN in 1.5500 sec 2024-11-15T00:46:11,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T00:46:11,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5540 sec 2024-11-15T00:46:11,969 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631571969"}]},"ts":"1731631571969"} 2024-11-15T00:46:11,970 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:46:11,970 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:46:11,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5650 sec 2024-11-15T00:46:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:46:12,560 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:12,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:46:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:12,562 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T00:46:12,562 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:12,565 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:12,566 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/recovered.edits] 2024-11-15T00:46:12,569 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2028bc5b85424b57bf83f4b0f2fcc549 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/2028bc5b85424b57bf83f4b0f2fcc549 2024-11-15T00:46:12,570 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/5ed0f05f28bd46d6833d5f7fa4f495fe to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/A/5ed0f05f28bd46d6833d5f7fa4f495fe 2024-11-15T00:46:12,572 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a9f32d3dbb884d0588dea7f6c7ca9d09 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/a9f32d3dbb884d0588dea7f6c7ca9d09 2024-11-15T00:46:12,573 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/dbf07671495443e0b863cea39cc5e0cd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/B/dbf07671495443e0b863cea39cc5e0cd 2024-11-15T00:46:12,575 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/44d073f7936d44d187ac95961ac1ff10 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/44d073f7936d44d187ac95961ac1ff10 2024-11-15T00:46:12,576 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/b195029dbbdf435bbb74667e8684d3cb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/C/b195029dbbdf435bbb74667e8684d3cb 2024-11-15T00:46:12,578 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/recovered.edits/656.seqid to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb/recovered.edits/656.seqid 2024-11-15T00:46:12,579 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/12c7a62476f136ecef48b9c405ac1dfb 2024-11-15T00:46:12,579 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:46:12,581 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:12,583 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:46:12,585 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:46:12,586 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:12,586 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:46:12,587 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631572586"}]},"ts":"9223372036854775807"} 2024-11-15T00:46:12,588 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:46:12,588 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 12c7a62476f136ecef48b9c405ac1dfb, NAME => 'TestAcidGuarantees,,1731631545510.12c7a62476f136ecef48b9c405ac1dfb.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:46:12,589 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:46:12,589 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631572589"}]},"ts":"9223372036854775807"} 2024-11-15T00:46:12,590 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:46:12,591 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:12,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-15T00:46:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T00:46:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:46:12,670 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:12,677 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=253 (was 248) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1639430675_22 at /127.0.0.1:58500 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1114251894_22 at /127.0.0.1:50372 [Waiting for operation #245] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1114251894_22 at /127.0.0.1:37572 [Waiting for operation #432] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1639430675_22 at /127.0.0.1:58516 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1114251894_22 at /127.0.0.1:58512 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1069 (was 1022) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1506 (was 959) - AvailableMemoryMB LEAK? - 2024-11-15T00:46:12,686 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=253, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=1069, ProcessCount=15, AvailableMemoryMB=1503 2024-11-15T00:46:12,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:46:12,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:46:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:12,693 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:46:12,694 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:12,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-11-15T00:46:12,695 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:46:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-15T00:46:12,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742143_1319 (size=1061) 2024-11-15T00:46:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-15T00:46:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-15T00:46:13,117 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403 2024-11-15T00:46:13,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742144_1320 (size=53) 2024-11-15T00:46:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-15T00:46:13,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:46:13,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ccd4c1fbc17fcc85146b8fbdf8bdc2ff, disabling compactions & flushes 2024-11-15T00:46:13,539 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:13,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:13,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. after waiting 0 ms 2024-11-15T00:46:13,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:13,539 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:13,539 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: Waiting for close lock at 1731631573539Disabling compacts and flushes for region at 1731631573539Disabling writes for close at 1731631573539Writing region close event to WAL at 1731631573539Closed at 1731631573539 2024-11-15T00:46:13,540 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:46:13,540 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631573540"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631573540"}]},"ts":"1731631573540"} 2024-11-15T00:46:13,542 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:46:13,543 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:46:13,543 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631573543"}]},"ts":"1731631573543"} 2024-11-15T00:46:13,544 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:46:13,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, ASSIGN}] 2024-11-15T00:46:13,545 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, ASSIGN 2024-11-15T00:46:13,546 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,34537,1731631472208; forceNewPlan=false, retain=false 2024-11-15T00:46:13,697 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, regionState=OPENING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:13,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, ASSIGN because future has completed 2024-11-15T00:46:13,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:46:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-15T00:46:13,854 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:13,854 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:46:13,854 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,854 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:46:13,854 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,854 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,856 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,857 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:46:13,858 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd4c1fbc17fcc85146b8fbdf8bdc2ff columnFamilyName A 2024-11-15T00:46:13,858 DEBUG [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:13,858 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(327): Store=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:46:13,858 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,859 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:46:13,859 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd4c1fbc17fcc85146b8fbdf8bdc2ff columnFamilyName B 2024-11-15T00:46:13,859 DEBUG [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:13,860 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(327): Store=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:46:13,860 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,861 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:46:13,861 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd4c1fbc17fcc85146b8fbdf8bdc2ff columnFamilyName C 2024-11-15T00:46:13,861 DEBUG [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:13,862 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(327): Store=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:46:13,862 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:13,862 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,863 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,863 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,864 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,864 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,864 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:46:13,866 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,868 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:46:13,868 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened ccd4c1fbc17fcc85146b8fbdf8bdc2ff; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59277431, jitterRate=-0.11669744551181793}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:46:13,868 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:13,869 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: Running coprocessor pre-open hook at 1731631573855Writing region info on filesystem at 1731631573855Initializing all the Stores at 1731631573855Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631573856 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631573856Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631573856Cleaning up temporary data from old regions at 1731631573864 (+8 ms)Running coprocessor post-open hooks at 1731631573868 (+4 ms)Region opened successfully at 1731631573869 (+1 ms) 2024-11-15T00:46:13,870 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., pid=83, masterSystemTime=1731631573851 2024-11-15T00:46:13,872 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:13,872 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:13,873 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:13,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:46:13,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-15T00:46:13,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 in 175 msec 2024-11-15T00:46:13,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-15T00:46:13,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, ASSIGN in 334 msec 2024-11-15T00:46:13,881 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:46:13,881 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631573881"}]},"ts":"1731631573881"} 2024-11-15T00:46:13,882 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:46:13,883 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:46:13,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1960 sec 2024-11-15T00:46:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-15T00:46:14,849 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:14,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:14,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:14,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:14,852 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685dc8d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:14,854 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208ea56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:14,855 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:14,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:14,858 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:46:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,33521,1731631471305 2024-11-15T00:46:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3a4c5e 2024-11-15T00:46:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:46:14,861 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:46:14,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:46:14,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:46:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:14,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742145_1321 (size=1097) 2024-11-15T00:46:14,891 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T00:46:14,891 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T00:46:14,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:46:14,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, REOPEN/MOVE}] 2024-11-15T00:46:14,901 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, REOPEN/MOVE 2024-11-15T00:46:14,902 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, regionState=CLOSING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:14,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, REOPEN/MOVE because future has completed 2024-11-15T00:46:14,905 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:46:14,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:46:15,060 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,060 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:46:15,060 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing ccd4c1fbc17fcc85146b8fbdf8bdc2ff, disabling compactions & flushes 2024-11-15T00:46:15,060 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,060 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,060 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. after waiting 0 ms 2024-11-15T00:46:15,060 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,070 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:46:15,070 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,070 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: Waiting for close lock at 1731631575060Running coprocessor pre-close hooks at 1731631575060Disabling compacts and flushes for region at 1731631575060Disabling writes for close at 1731631575060Writing region close event to WAL at 1731631575061 (+1 ms)Running coprocessor post-close hooks at 1731631575070 (+9 ms)Closed at 1731631575070 2024-11-15T00:46:15,070 WARN [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: ccd4c1fbc17fcc85146b8fbdf8bdc2ff to self. 2024-11-15T00:46:15,072 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,073 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, regionState=CLOSED 2024-11-15T00:46:15,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:46:15,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-15T00:46:15,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 in 170 msec 2024-11-15T00:46:15,078 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, REOPEN/MOVE; state=CLOSED, location=fb2f6cb7846a,34537,1731631472208; forceNewPlan=false, retain=true 2024-11-15T00:46:15,229 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, regionState=OPENING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, REOPEN/MOVE because future has completed 2024-11-15T00:46:15,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:46:15,386 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,386 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:46:15,386 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,386 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:46:15,386 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,386 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,388 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,388 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:46:15,389 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd4c1fbc17fcc85146b8fbdf8bdc2ff columnFamilyName A 2024-11-15T00:46:15,390 DEBUG [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:15,390 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(327): Store=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:46:15,390 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,391 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:46:15,391 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd4c1fbc17fcc85146b8fbdf8bdc2ff columnFamilyName B 2024-11-15T00:46:15,391 DEBUG [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:15,392 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(327): Store=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:46:15,392 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,392 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:46:15,392 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd4c1fbc17fcc85146b8fbdf8bdc2ff columnFamilyName C 2024-11-15T00:46:15,392 DEBUG [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:15,393 INFO [StoreOpener-ccd4c1fbc17fcc85146b8fbdf8bdc2ff-1 {}] regionserver.HStore(327): Store=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:46:15,393 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,393 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,394 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,394 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,395 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,395 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,396 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:46:15,397 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,398 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened ccd4c1fbc17fcc85146b8fbdf8bdc2ff; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75141451, jitterRate=0.11969487369060516}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:46:15,398 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,399 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: Running coprocessor pre-open hook at 1731631575386Writing region info on filesystem at 1731631575386Initializing all the Stores at 1731631575387 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631575387Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631575387Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631575388 (+1 ms)Cleaning up temporary data from old regions at 1731631575395 (+7 ms)Running coprocessor post-open hooks at 1731631575398 (+3 ms)Region opened successfully at 1731631575398 2024-11-15T00:46:15,399 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., pid=88, masterSystemTime=1731631575383 2024-11-15T00:46:15,401 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,401 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,401 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, regionState=OPEN, openSeqNum=5, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:46:15,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-11-15T00:46:15,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 in 172 msec 2024-11-15T00:46:15,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-15T00:46:15,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, REOPEN/MOVE in 505 msec 2024-11-15T00:46:15,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-15T00:46:15,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 512 msec 2024-11-15T00:46:15,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 546 msec 2024-11-15T00:46:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T00:46:15,411 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:15,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:15,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:15,413 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a75e35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:15,414 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d7d690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:15,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:15,416 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6090972d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:15,418 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:15,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bcbfa89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:15,419 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:15,420 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6f86d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:15,421 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ab4bee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:15,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:15,422 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:15,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:15,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:15,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d2aef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:15,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:15,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:15,424 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@131078cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:15,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:15,425 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c41b1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:15,426 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@270eb851, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:15,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:15,428 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a582b55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:15,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:15,429 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327720d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:15,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:15,430 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac09d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:15,432 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aeae733, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:15,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:15,433 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6824aa42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:15,434 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:15,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c84c3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:15,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:15,436 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218333e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:15,437 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:15,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70aa01b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:15,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:15,439 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:15,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:15,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3549b5c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:15,441 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1ca225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:15,442 DEBUG [Thread-1496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:15,442 DEBUG [Thread-1494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:15,442 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:15,442 DEBUG [Thread-1492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:15,442 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:15,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:15,442 DEBUG [Thread-1490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:15,443 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:15,443 DEBUG [Thread-1479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-15T00:46:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:15,444 DEBUG [Thread-1490 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:15,444 DEBUG [Thread-1479 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:15,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T00:46:15,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:15,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:15,445 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:15,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:15,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:15,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:15,446 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:15,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:15,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:15,447 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:46:15,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:46:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:46:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:46:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:46:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:46:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:46:15,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:46:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:46:15,456 DEBUG [Thread-1498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:15,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:15,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:15,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:46:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T00:46:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:15,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150875da7e1c424344a00c90429b2fe7d5_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631575458/Put/seqid=0 2024-11-15T00:46:15,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631635495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631635496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631635498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631635500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742146_1322 (size=9714) 2024-11-15T00:46:15,510 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:15,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631635502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,516 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150875da7e1c424344a00c90429b2fe7d5_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150875da7e1c424344a00c90429b2fe7d5_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:15,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/2049c710453e4d1ea49af3492af1619f, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:15,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/2049c710453e4d1ea49af3492af1619f is 175, key is test_row_0/A:col10/1731631575458/Put/seqid=0 2024-11-15T00:46:15,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742147_1323 (size=22361) 2024-11-15T00:46:15,546 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/2049c710453e4d1ea49af3492af1619f 2024-11-15T00:46:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T00:46:15,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/bdf76800a989406786f7400ce6a4b762 is 50, key is test_row_0/B:col10/1731631575458/Put/seqid=0 2024-11-15T00:46:15,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742148_1324 (size=9657) 2024-11-15T00:46:15,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:15,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:15,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:15,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:15,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631635609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631635609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631635619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631635619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631635619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:15,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:15,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:15,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T00:46:15,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631635819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631635820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631635830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631635830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631635830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:15,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:15,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:15,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:15,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:15,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:15,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/bdf76800a989406786f7400ce6a4b762 2024-11-15T00:46:16,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/cdc733889dc947bbb86f72f0b016d211 is 50, key is test_row_0/C:col10/1731631575458/Put/seqid=0 2024-11-15T00:46:16,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742149_1325 (size=9657) 2024-11-15T00:46:16,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/cdc733889dc947bbb86f72f0b016d211 2024-11-15T00:46:16,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/2049c710453e4d1ea49af3492af1619f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2049c710453e4d1ea49af3492af1619f 2024-11-15T00:46:16,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T00:46:16,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2049c710453e4d1ea49af3492af1619f, entries=100, sequenceid=16, filesize=21.8 K 2024-11-15T00:46:16,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/bdf76800a989406786f7400ce6a4b762 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bdf76800a989406786f7400ce6a4b762 2024-11-15T00:46:16,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bdf76800a989406786f7400ce6a4b762, entries=100, sequenceid=16, filesize=9.4 K 2024-11-15T00:46:16,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/cdc733889dc947bbb86f72f0b016d211 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/cdc733889dc947bbb86f72f0b016d211 2024-11-15T00:46:16,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/cdc733889dc947bbb86f72f0b016d211, entries=100, sequenceid=16, filesize=9.4 K 2024-11-15T00:46:16,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 632ms, sequenceid=16, compaction requested=false 2024-11-15T00:46:16,093 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T00:46:16,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:16,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T00:46:16,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631575483, max timestamp=1731631576144Num uniques 859; 2024-11-15T00:46:16,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631575483, max timestamp=1731631576144Num uniques 859; 2024-11-15T00:46:16,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631575483, max timestamp=1731631576144Num uniques 859; 2024-11-15T00:46:16,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:46:16,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:46:16,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:46:16,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:16,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:16,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:16,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:16,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:16,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:16,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111513177b499d44483baabd33797ebc62b0_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631575498/Put/seqid=0 2024-11-15T00:46:16,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742150_1326 (size=12154) 2024-11-15T00:46:16,182 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:16,191 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111513177b499d44483baabd33797ebc62b0_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111513177b499d44483baabd33797ebc62b0_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:16,194 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/9dc40ff70e434498b0807bbb0d09d28b, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:16,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/9dc40ff70e434498b0807bbb0d09d28b is 175, key is test_row_0/A:col10/1731631575498/Put/seqid=0 2024-11-15T00:46:16,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631576179, max timestamp=1731631576199Num uniques 859; 2024-11-15T00:46:16,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:16,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631576179, max timestamp=1731631576199Num uniques 859; 2024-11-15T00:46:16,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631576179, max timestamp=1731631576199Num uniques 859; 2024-11-15T00:46:16,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:16,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:16,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742151_1327 (size=30955) 2024-11-15T00:46:16,226 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=38, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/9dc40ff70e434498b0807bbb0d09d28b 2024-11-15T00:46:16,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/c64c7abb3f8d4402b03a77d38fdef28c is 50, key is test_row_0/B:col10/1731631575498/Put/seqid=0 2024-11-15T00:46:16,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742152_1328 (size=12001) 2024-11-15T00:46:16,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/c64c7abb3f8d4402b03a77d38fdef28c 2024-11-15T00:46:16,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731631576238, max timestamp=1731631576267Num uniques 1068; 2024-11-15T00:46:16,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-15T00:46:16,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/08a51efbe11e46c78039d52246abb3ee is 50, key is test_row_0/C:col10/1731631575498/Put/seqid=0 2024-11-15T00:46:16,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731631576238, max timestamp=1731631576267Num uniques 1068; 2024-11-15T00:46:16,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-15T00:46:16,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731631576238, max timestamp=1731631576267Num uniques 1068; 2024-11-15T00:46:16,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742153_1329 (size=12001) 2024-11-15T00:46:16,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-15T00:46:16,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/08a51efbe11e46c78039d52246abb3ee 2024-11-15T00:46:16,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/9dc40ff70e434498b0807bbb0d09d28b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9dc40ff70e434498b0807bbb0d09d28b 2024-11-15T00:46:16,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9dc40ff70e434498b0807bbb0d09d28b, entries=150, sequenceid=38, filesize=30.2 K 2024-11-15T00:46:16,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/c64c7abb3f8d4402b03a77d38fdef28c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/c64c7abb3f8d4402b03a77d38fdef28c 2024-11-15T00:46:16,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/c64c7abb3f8d4402b03a77d38fdef28c, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T00:46:16,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/08a51efbe11e46c78039d52246abb3ee as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/08a51efbe11e46c78039d52246abb3ee 2024-11-15T00:46:16,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:16,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:16,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731631576333, max timestamp=1731631576381Num uniques 1127; 2024-11-15T00:46:16,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731631576333, max timestamp=1731631576381Num uniques 1127; 2024-11-15T00:46:16,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-15T00:46:16,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-15T00:46:16,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/08a51efbe11e46c78039d52246abb3ee, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T00:46:16,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=84.98 KB/87020 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 243ms, sequenceid=38, compaction requested=false 2024-11-15T00:46:16,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:16,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731631576333, max timestamp=1731631576381Num uniques 1127; 2024-11-15T00:46:16,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-15T00:46:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:16,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=51.44 KB heapSize=129.78 KB 2024-11-15T00:46:16,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:16,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:16,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:16,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:16,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:16,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:16,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157997b811ddb04b2f81e20c1514a96e83_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631576373/Put/seqid=0 2024-11-15T00:46:16,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742154_1330 (size=21918) 2024-11-15T00:46:16,476 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:16,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631576422, max timestamp=1731631576497Num uniques 859; 2024-11-15T00:46:16,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:46:16,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631576422, max timestamp=1731631576497Num uniques 859; 2024-11-15T00:46:16,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:46:16,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631576422, max timestamp=1731631576497Num uniques 859; 2024-11-15T00:46:16,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:46:16,534 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157997b811ddb04b2f81e20c1514a96e83_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157997b811ddb04b2f81e20c1514a96e83_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:16,542 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/354ce60027c14c51901357412155f22e, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:16,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/354ce60027c14c51901357412155f22e is 175, key is test_row_0/A:col10/1731631576373/Put/seqid=0 2024-11-15T00:46:16,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:16,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:16,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742155_1331 (size=65323) 2024-11-15T00:46:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T00:46:16,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631636566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631636569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631636584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631636586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631636587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631636689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631636689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631636700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631636709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631636709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:16,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:16,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631636909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631636909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631636910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631636929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631636932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:16,981 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/354ce60027c14c51901357412155f22e 2024-11-15T00:46:17,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/bf46fa6f96ca42149460fae9d5a7856b is 50, key is test_row_0/B:col10/1731631576373/Put/seqid=0 2024-11-15T00:46:17,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742156_1332 (size=12001) 2024-11-15T00:46:17,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/bf46fa6f96ca42149460fae9d5a7856b 2024-11-15T00:46:17,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/aceb50aa1e9a409ea903eee29abf888b is 50, key is test_row_0/C:col10/1731631576373/Put/seqid=0 2024-11-15T00:46:17,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:17,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:17,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742157_1333 (size=12001) 2024-11-15T00:46:17,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/aceb50aa1e9a409ea903eee29abf888b 2024-11-15T00:46:17,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/354ce60027c14c51901357412155f22e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/354ce60027c14c51901357412155f22e 2024-11-15T00:46:17,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/354ce60027c14c51901357412155f22e, entries=350, sequenceid=95, filesize=63.8 K 2024-11-15T00:46:17,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/bf46fa6f96ca42149460fae9d5a7856b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bf46fa6f96ca42149460fae9d5a7856b 2024-11-15T00:46:17,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bf46fa6f96ca42149460fae9d5a7856b, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T00:46:17,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/aceb50aa1e9a409ea903eee29abf888b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/aceb50aa1e9a409ea903eee29abf888b 2024-11-15T00:46:17,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/aceb50aa1e9a409ea903eee29abf888b, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T00:46:17,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.44 KB/52670, heapSize ~129.73 KB/132848, currentSize=140.89 KB/144270 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 728ms, sequenceid=95, compaction requested=true 2024-11-15T00:46:17,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:17,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:17,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:17,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:17,121 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:17,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:17,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:17,121 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:17,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:17,123 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:17,123 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B is initiating minor compaction (all files) 2024-11-15T00:46:17,123 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:17,123 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bdf76800a989406786f7400ce6a4b762, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/c64c7abb3f8d4402b03a77d38fdef28c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bf46fa6f96ca42149460fae9d5a7856b] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=32.9 K 2024-11-15T00:46:17,129 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:17,129 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A is initiating minor compaction (all files) 2024-11-15T00:46:17,129 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:17,129 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2049c710453e4d1ea49af3492af1619f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9dc40ff70e434498b0807bbb0d09d28b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/354ce60027c14c51901357412155f22e] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=115.9 K 2024-11-15T00:46:17,129 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:17,129 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2049c710453e4d1ea49af3492af1619f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9dc40ff70e434498b0807bbb0d09d28b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/354ce60027c14c51901357412155f22e] 2024-11-15T00:46:17,130 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf76800a989406786f7400ce6a4b762, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631575458 2024-11-15T00:46:17,131 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c64c7abb3f8d4402b03a77d38fdef28c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731631575483 2024-11-15T00:46:17,131 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting bf46fa6f96ca42149460fae9d5a7856b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631576333 2024-11-15T00:46:17,131 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2049c710453e4d1ea49af3492af1619f, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631575458 2024-11-15T00:46:17,132 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dc40ff70e434498b0807bbb0d09d28b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731631575483 2024-11-15T00:46:17,133 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 354ce60027c14c51901357412155f22e, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631576333 2024-11-15T00:46:17,150 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#B#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:17,151 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/2c1c5663cb0d413ab103c8946ce3f593 is 50, key is test_row_0/B:col10/1731631576373/Put/seqid=0 2024-11-15T00:46:17,167 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:17,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-15T00:46:17,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:17,181 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-15T00:46:17,181 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115059ffba16b9e4521b2a62574ba87883e_ccd4c1fbc17fcc85146b8fbdf8bdc2ff store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:17,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:17,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:17,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:17,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:17,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:17,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:17,183 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115059ffba16b9e4521b2a62574ba87883e_ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:17,183 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115059ffba16b9e4521b2a62574ba87883e_ccd4c1fbc17fcc85146b8fbdf8bdc2ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:17,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742158_1334 (size=12104) 2024-11-15T00:46:17,212 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/2c1c5663cb0d413ab103c8946ce3f593 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2c1c5663cb0d413ab103c8946ce3f593 2024-11-15T00:46:17,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115586ce7abd2da4db6b74216080669b20d_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631576583/Put/seqid=0 2024-11-15T00:46:17,219 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 2c1c5663cb0d413ab103c8946ce3f593(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:17,219 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:17,219 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, priority=13, startTime=1731631577121; duration=0sec 2024-11-15T00:46:17,219 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:17,219 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:17,219 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:17,221 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:17,221 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C is initiating minor compaction (all files) 2024-11-15T00:46:17,221 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:17,221 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/cdc733889dc947bbb86f72f0b016d211, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/08a51efbe11e46c78039d52246abb3ee, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/aceb50aa1e9a409ea903eee29abf888b] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=32.9 K 2024-11-15T00:46:17,221 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting cdc733889dc947bbb86f72f0b016d211, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631575458 2024-11-15T00:46:17,221 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 08a51efbe11e46c78039d52246abb3ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731631575483 2024-11-15T00:46:17,222 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting aceb50aa1e9a409ea903eee29abf888b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631576333 2024-11-15T00:46:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:17,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:17,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742159_1335 (size=4469) 2024-11-15T00:46:17,255 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#A#compaction#285 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:17,256 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/72359b737ffc44b099ea9de0c302f820 is 175, key is test_row_0/A:col10/1731631576373/Put/seqid=0 2024-11-15T00:46:17,264 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:17,265 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/4e18fb049bb64cb9aa3694db9306ce20 is 50, key is test_row_0/C:col10/1731631576373/Put/seqid=0 2024-11-15T00:46:17,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631637249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631637250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631637268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742160_1336 (size=12204) 2024-11-15T00:46:17,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631637273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631637274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742161_1337 (size=31058) 2024-11-15T00:46:17,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742162_1338 (size=12104) 2024-11-15T00:46:17,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631637379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631637379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631637399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631637409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,478 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:46:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T00:46:17,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631637599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631637599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631637619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631637630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:17,690 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115586ce7abd2da4db6b74216080669b20d_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115586ce7abd2da4db6b74216080669b20d_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:17,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/173f9ca917ab450a8b4d01e9fa0cf63c, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:17,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/173f9ca917ab450a8b4d01e9fa0cf63c is 175, key is test_row_0/A:col10/1731631576583/Put/seqid=0 2024-11-15T00:46:17,710 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/72359b737ffc44b099ea9de0c302f820 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/72359b737ffc44b099ea9de0c302f820 2024-11-15T00:46:17,725 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 72359b737ffc44b099ea9de0c302f820(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:17,725 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:17,725 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, priority=13, startTime=1731631577121; duration=0sec 2024-11-15T00:46:17,725 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:17,725 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:17,727 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/4e18fb049bb64cb9aa3694db9306ce20 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/4e18fb049bb64cb9aa3694db9306ce20 2024-11-15T00:46:17,738 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 4e18fb049bb64cb9aa3694db9306ce20(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:17,738 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:17,738 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, priority=13, startTime=1731631577121; duration=0sec 2024-11-15T00:46:17,738 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:17,738 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:17,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742163_1339 (size=31005) 2024-11-15T00:46:17,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631637779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631637920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631637920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631637939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:17,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631637949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,147 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/173f9ca917ab450a8b4d01e9fa0cf63c 2024-11-15T00:46:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/971ad0a7e27544b2a396c78710780c4e is 50, key is test_row_0/B:col10/1731631576583/Put/seqid=0 2024-11-15T00:46:18,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742164_1340 (size=12051) 2024-11-15T00:46:18,187 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/971ad0a7e27544b2a396c78710780c4e 2024-11-15T00:46:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/924d7792fd864d3790f4ef9a787c9cb3 is 50, key is test_row_0/C:col10/1731631576583/Put/seqid=0 2024-11-15T00:46:18,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742165_1341 (size=12051) 2024-11-15T00:46:18,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/924d7792fd864d3790f4ef9a787c9cb3 2024-11-15T00:46:18,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/173f9ca917ab450a8b4d01e9fa0cf63c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/173f9ca917ab450a8b4d01e9fa0cf63c 2024-11-15T00:46:18,262 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/173f9ca917ab450a8b4d01e9fa0cf63c, entries=150, sequenceid=131, filesize=30.3 K 2024-11-15T00:46:18,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/971ad0a7e27544b2a396c78710780c4e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/971ad0a7e27544b2a396c78710780c4e 2024-11-15T00:46:18,272 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/971ad0a7e27544b2a396c78710780c4e, entries=150, sequenceid=131, filesize=11.8 K 2024-11-15T00:46:18,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/924d7792fd864d3790f4ef9a787c9cb3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/924d7792fd864d3790f4ef9a787c9cb3 2024-11-15T00:46:18,282 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/924d7792fd864d3790f4ef9a787c9cb3, entries=150, sequenceid=131, filesize=11.8 K 2024-11-15T00:46:18,283 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 1102ms, sequenceid=131, compaction requested=false 2024-11-15T00:46:18,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:18,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:18,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-15T00:46:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-15T00:46:18,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-15T00:46:18,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8430 sec 2024-11-15T00:46:18,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.8520 sec 2024-11-15T00:46:18,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T00:46:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:18,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:18,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:18,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:18,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:18,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:18,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:18,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155df4233683664b909a0c8067502780a0_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631578453/Put/seqid=0 2024-11-15T00:46:18,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742166_1342 (size=14794) 2024-11-15T00:46:18,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:18,535 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155df4233683664b909a0c8067502780a0_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155df4233683664b909a0c8067502780a0_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:18,537 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/2a8d4345301445cd86d6cf7dbd01245d, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:18,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/2a8d4345301445cd86d6cf7dbd01245d is 175, key is test_row_0/A:col10/1731631578453/Put/seqid=0 2024-11-15T00:46:18,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742167_1343 (size=39749) 2024-11-15T00:46:18,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631638574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631638577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631638580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631638603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631638719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631638719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631638719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631638732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631638800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,807 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in ccd4c1fbc17fcc85146b8fbdf8bdc2ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631638939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631638939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631638939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,955 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/2a8d4345301445cd86d6cf7dbd01245d 2024-11-15T00:46:18,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/3576d8b7d36247f0a7a8c7b180a136f6 is 50, key is test_row_0/B:col10/1731631578453/Put/seqid=0 2024-11-15T00:46:18,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:18,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742168_1344 (size=12151) 2024-11-15T00:46:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631638959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:18,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/3576d8b7d36247f0a7a8c7b180a136f6 2024-11-15T00:46:18,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/53e9e007b3f54b3d84db014fec2ebcec is 50, key is test_row_0/C:col10/1731631578453/Put/seqid=0 2024-11-15T00:46:18,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742169_1345 (size=12151) 2024-11-15T00:46:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631639249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631639249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631639250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:19,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631639279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:19,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/53e9e007b3f54b3d84db014fec2ebcec 2024-11-15T00:46:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/2a8d4345301445cd86d6cf7dbd01245d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2a8d4345301445cd86d6cf7dbd01245d 2024-11-15T00:46:19,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2a8d4345301445cd86d6cf7dbd01245d, entries=200, sequenceid=146, filesize=38.8 K 2024-11-15T00:46:19,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/3576d8b7d36247f0a7a8c7b180a136f6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3576d8b7d36247f0a7a8c7b180a136f6 2024-11-15T00:46:19,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3576d8b7d36247f0a7a8c7b180a136f6, entries=150, sequenceid=146, filesize=11.9 K 2024-11-15T00:46:19,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/53e9e007b3f54b3d84db014fec2ebcec as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/53e9e007b3f54b3d84db014fec2ebcec 2024-11-15T00:46:19,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/53e9e007b3f54b3d84db014fec2ebcec, entries=150, sequenceid=146, filesize=11.9 K 2024-11-15T00:46:19,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 954ms, sequenceid=146, compaction requested=true 2024-11-15T00:46:19,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:19,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:19,410 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:19,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:19,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:19,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:19,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:19,410 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:19,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:19,411 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101812 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:19,411 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A is initiating minor compaction (all files) 2024-11-15T00:46:19,411 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:19,411 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/72359b737ffc44b099ea9de0c302f820, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/173f9ca917ab450a8b4d01e9fa0cf63c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2a8d4345301445cd86d6cf7dbd01245d] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=99.4 K 2024-11-15T00:46:19,411 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:19,411 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/72359b737ffc44b099ea9de0c302f820, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/173f9ca917ab450a8b4d01e9fa0cf63c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2a8d4345301445cd86d6cf7dbd01245d] 2024-11-15T00:46:19,411 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36306 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:19,412 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B is initiating minor compaction (all files) 2024-11-15T00:46:19,412 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:19,412 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2c1c5663cb0d413ab103c8946ce3f593, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/971ad0a7e27544b2a396c78710780c4e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3576d8b7d36247f0a7a8c7b180a136f6] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=35.5 K 2024-11-15T00:46:19,412 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c1c5663cb0d413ab103c8946ce3f593, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631576333 2024-11-15T00:46:19,412 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72359b737ffc44b099ea9de0c302f820, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631576333 2024-11-15T00:46:19,412 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 971ad0a7e27544b2a396c78710780c4e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731631576566 2024-11-15T00:46:19,412 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 173f9ca917ab450a8b4d01e9fa0cf63c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731631576566 2024-11-15T00:46:19,412 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3576d8b7d36247f0a7a8c7b180a136f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731631577248 2024-11-15T00:46:19,412 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a8d4345301445cd86d6cf7dbd01245d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731631577248 2024-11-15T00:46:19,417 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:19,418 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e96fd031e49a46c4944ca956614e8b74_ccd4c1fbc17fcc85146b8fbdf8bdc2ff store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:19,418 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#B#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:19,419 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/96e51a4824fa4d0583bc408b7d3ab741 is 50, key is test_row_0/B:col10/1731631578453/Put/seqid=0 2024-11-15T00:46:19,420 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e96fd031e49a46c4944ca956614e8b74_ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:19,420 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e96fd031e49a46c4944ca956614e8b74_ccd4c1fbc17fcc85146b8fbdf8bdc2ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:19,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742171_1347 (size=4469) 2024-11-15T00:46:19,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742170_1346 (size=12357) 2024-11-15T00:46:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-15T00:46:19,610 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:19,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-15T00:46:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T00:46:19,612 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:19,613 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:19,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T00:46:19,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-15T00:46:19,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:19,766 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:46:19,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:19,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:19,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:19,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:19,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:19,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:19,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:19,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fdcd99b8fe0a4d3ca8e13bdc5c46ad0d_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631578537/Put/seqid=0 2024-11-15T00:46:19,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742172_1348 (size=12304) 2024-11-15T00:46:19,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631639799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631639800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:19,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631639818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:19,826 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#A#compaction#294 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:19,827 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/f3faf79e7cd044d2bfa46c15d5c7acda is 175, key is test_row_0/A:col10/1731631578453/Put/seqid=0 2024-11-15T00:46:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631639820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:19,847 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/96e51a4824fa4d0583bc408b7d3ab741 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/96e51a4824fa4d0583bc408b7d3ab741 2024-11-15T00:46:19,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742173_1349 (size=31311) 2024-11-15T00:46:19,855 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/f3faf79e7cd044d2bfa46c15d5c7acda as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f3faf79e7cd044d2bfa46c15d5c7acda 2024-11-15T00:46:19,857 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 96e51a4824fa4d0583bc408b7d3ab741(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:19,857 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:19,857 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, priority=13, startTime=1731631579410; duration=0sec 2024-11-15T00:46:19,858 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:19,858 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:19,858 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:19,862 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36306 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:19,862 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C is initiating minor compaction (all files) 2024-11-15T00:46:19,862 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:19,862 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/4e18fb049bb64cb9aa3694db9306ce20, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/924d7792fd864d3790f4ef9a787c9cb3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/53e9e007b3f54b3d84db014fec2ebcec] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=35.5 K 2024-11-15T00:46:19,862 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e18fb049bb64cb9aa3694db9306ce20, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631576333 2024-11-15T00:46:19,863 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 924d7792fd864d3790f4ef9a787c9cb3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731631576566 2024-11-15T00:46:19,863 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e9e007b3f54b3d84db014fec2ebcec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731631577248 2024-11-15T00:46:19,868 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into f3faf79e7cd044d2bfa46c15d5c7acda(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:19,868 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:19,868 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, priority=13, startTime=1731631579410; duration=0sec 2024-11-15T00:46:19,868 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:19,868 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:19,878 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:19,879 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/242a3525ea4046cd8841614dfae8b065 is 50, key is test_row_0/C:col10/1731631578453/Put/seqid=0 2024-11-15T00:46:19,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742174_1350 (size=12357) 2024-11-15T00:46:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T00:46:19,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631639930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631639930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:19,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631639939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631640149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631640149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631640149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:20,218 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fdcd99b8fe0a4d3ca8e13bdc5c46ad0d_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fdcd99b8fe0a4d3ca8e13bdc5c46ad0d_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:20,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/be6eb3e3a37842109fef8d7b670bf05a, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:20,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/be6eb3e3a37842109fef8d7b670bf05a is 175, key is test_row_0/A:col10/1731631578537/Put/seqid=0 2024-11-15T00:46:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T00:46:20,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742175_1351 (size=31105) 2024-11-15T00:46:20,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/be6eb3e3a37842109fef8d7b670bf05a 2024-11-15T00:46:20,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/63061d01510b4dcfab0e1d27a9d54b34 is 50, key is test_row_0/B:col10/1731631578537/Put/seqid=0 2024-11-15T00:46:20,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742176_1352 (size=12151) 2024-11-15T00:46:20,311 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/63061d01510b4dcfab0e1d27a9d54b34 2024-11-15T00:46:20,329 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/242a3525ea4046cd8841614dfae8b065 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/242a3525ea4046cd8841614dfae8b065 2024-11-15T00:46:20,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/272a61d92081406da4e0f59ffd4e1b84 is 50, key is test_row_0/C:col10/1731631578537/Put/seqid=0 2024-11-15T00:46:20,351 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 242a3525ea4046cd8841614dfae8b065(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:20,351 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:20,351 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, priority=13, startTime=1731631579410; duration=0sec 2024-11-15T00:46:20,351 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:20,351 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:20,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742177_1353 (size=12151) 2024-11-15T00:46:20,364 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/272a61d92081406da4e0f59ffd4e1b84 2024-11-15T00:46:20,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/be6eb3e3a37842109fef8d7b670bf05a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/be6eb3e3a37842109fef8d7b670bf05a 2024-11-15T00:46:20,390 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/be6eb3e3a37842109fef8d7b670bf05a, entries=150, sequenceid=166, filesize=30.4 K 2024-11-15T00:46:20,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/63061d01510b4dcfab0e1d27a9d54b34 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/63061d01510b4dcfab0e1d27a9d54b34 2024-11-15T00:46:20,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/63061d01510b4dcfab0e1d27a9d54b34, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T00:46:20,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/272a61d92081406da4e0f59ffd4e1b84 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/272a61d92081406da4e0f59ffd4e1b84 2024-11-15T00:46:20,441 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/272a61d92081406da4e0f59ffd4e1b84, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T00:46:20,442 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 676ms, sequenceid=166, compaction requested=false 2024-11-15T00:46:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-15T00:46:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-15T00:46:20,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-15T00:46:20,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 831 msec 2024-11-15T00:46:20,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 838 msec 2024-11-15T00:46:20,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T00:46:20,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:20,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:20,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:20,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:20,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:20,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:20,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ae39b522cc354668ab855fadcac56bab_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631580482/Put/seqid=0 2024-11-15T00:46:20,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742178_1354 (size=12304) 2024-11-15T00:46:20,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631640564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631640567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631640575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631640690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631640690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631640690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T00:46:20,761 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:20,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-15T00:46:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T00:46:20,764 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:20,765 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:20,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:20,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631640836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,843 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in ccd4c1fbc17fcc85146b8fbdf8bdc2ff of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631640839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,853 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in ccd4c1fbc17fcc85146b8fbdf8bdc2ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T00:46:20,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631640899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631640899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631640899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:20,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T00:46:20,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:20,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:20,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:20,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:20,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:20,986 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:21,002 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ae39b522cc354668ab855fadcac56bab_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ae39b522cc354668ab855fadcac56bab_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:21,005 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/acb789889e8c468ca8cdbbd571fc9faf, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:21,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/acb789889e8c468ca8cdbbd571fc9faf is 175, key is test_row_0/A:col10/1731631580482/Put/seqid=0 2024-11-15T00:46:21,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742179_1355 (size=31105) 2024-11-15T00:46:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T00:46:21,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T00:46:21,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:21,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631641221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:21,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631641221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:21,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631641229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:21,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T00:46:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T00:46:21,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T00:46:21,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:21,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,432 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/acb789889e8c468ca8cdbbd571fc9faf 2024-11-15T00:46:21,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/f330d53c36174b7f83e75adbecd2ed7f is 50, key is test_row_0/B:col10/1731631580482/Put/seqid=0 2024-11-15T00:46:21,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742180_1356 (size=12151) 2024-11-15T00:46:21,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T00:46:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T00:46:21,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:21,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631641740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:21,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631641740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:21,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631641752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:21,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T00:46:21,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:21,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:21,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:21,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/f330d53c36174b7f83e75adbecd2ed7f 2024-11-15T00:46:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T00:46:21,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c894759d2c744551a9748a14323b40a0 is 50, key is test_row_0/C:col10/1731631580482/Put/seqid=0 2024-11-15T00:46:21,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742181_1357 (size=12151) 2024-11-15T00:46:21,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c894759d2c744551a9748a14323b40a0 2024-11-15T00:46:21,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/acb789889e8c468ca8cdbbd571fc9faf as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/acb789889e8c468ca8cdbbd571fc9faf 2024-11-15T00:46:21,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/acb789889e8c468ca8cdbbd571fc9faf, entries=150, sequenceid=184, filesize=30.4 K 2024-11-15T00:46:21,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/f330d53c36174b7f83e75adbecd2ed7f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f330d53c36174b7f83e75adbecd2ed7f 2024-11-15T00:46:21,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f330d53c36174b7f83e75adbecd2ed7f, entries=150, sequenceid=184, filesize=11.9 K 2024-11-15T00:46:21,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c894759d2c744551a9748a14323b40a0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c894759d2c744551a9748a14323b40a0 2024-11-15T00:46:22,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c894759d2c744551a9748a14323b40a0, entries=150, sequenceid=184, filesize=11.9 K 2024-11-15T00:46:22,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 1518ms, sequenceid=184, compaction requested=true 2024-11-15T00:46:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:22,006 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:22,006 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:22,011 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:22,011 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A is initiating minor compaction (all files) 2024-11-15T00:46:22,011 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:22,011 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f3faf79e7cd044d2bfa46c15d5c7acda, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/be6eb3e3a37842109fef8d7b670bf05a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/acb789889e8c468ca8cdbbd571fc9faf] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=91.3 K 2024-11-15T00:46:22,011 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:22,011 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f3faf79e7cd044d2bfa46c15d5c7acda, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/be6eb3e3a37842109fef8d7b670bf05a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/acb789889e8c468ca8cdbbd571fc9faf] 2024-11-15T00:46:22,012 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:22,012 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B is initiating minor compaction (all files) 2024-11-15T00:46:22,012 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:22,012 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/96e51a4824fa4d0583bc408b7d3ab741, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/63061d01510b4dcfab0e1d27a9d54b34, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f330d53c36174b7f83e75adbecd2ed7f] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=35.8 K 2024-11-15T00:46:22,012 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96e51a4824fa4d0583bc408b7d3ab741, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731631577248 2024-11-15T00:46:22,012 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f3faf79e7cd044d2bfa46c15d5c7acda, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731631577248 2024-11-15T00:46:22,013 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63061d01510b4dcfab0e1d27a9d54b34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631578537 2024-11-15T00:46:22,013 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting be6eb3e3a37842109fef8d7b670bf05a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631578537 2024-11-15T00:46:22,013 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting acb789889e8c468ca8cdbbd571fc9faf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631580480 2024-11-15T00:46:22,013 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting f330d53c36174b7f83e75adbecd2ed7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631580480 2024-11-15T00:46:22,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-15T00:46:22,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:22,024 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T00:46:22,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:22,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:22,047 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#B#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:22,048 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/f6907340a33746f1bdbe2734e38fe3a3 is 50, key is test_row_0/B:col10/1731631580482/Put/seqid=0 2024-11-15T00:46:22,049 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:22,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115eab9d3ee025440ca943d181b72a1dc00_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631580565/Put/seqid=0 2024-11-15T00:46:22,058 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411150133855c53ce46b3bee6bcdec40a9ae4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:22,061 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411150133855c53ce46b3bee6bcdec40a9ae4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:22,061 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150133855c53ce46b3bee6bcdec40a9ae4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:22,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742183_1359 (size=12459) 2024-11-15T00:46:22,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742182_1358 (size=12304) 2024-11-15T00:46:22,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:22,113 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115eab9d3ee025440ca943d181b72a1dc00_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115eab9d3ee025440ca943d181b72a1dc00_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/a097cdf9ee70446fb460d5f6645ebf0c, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:22,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/a097cdf9ee70446fb460d5f6645ebf0c is 175, key is test_row_0/A:col10/1731631580565/Put/seqid=0 2024-11-15T00:46:22,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742184_1360 (size=4469) 2024-11-15T00:46:22,151 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#A#compaction#303 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:22,151 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/626eba4d66894ec3b1277dd29ca5b71b is 175, key is test_row_0/A:col10/1731631580482/Put/seqid=0 2024-11-15T00:46:22,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742185_1361 (size=31105) 2024-11-15T00:46:22,158 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/a097cdf9ee70446fb460d5f6645ebf0c 2024-11-15T00:46:22,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/3c5e38c823c2401da30f22bf41a7a160 is 50, key is test_row_0/B:col10/1731631580565/Put/seqid=0 2024-11-15T00:46:22,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742186_1362 (size=31413) 2024-11-15T00:46:22,188 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/626eba4d66894ec3b1277dd29ca5b71b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/626eba4d66894ec3b1277dd29ca5b71b 2024-11-15T00:46:22,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742187_1363 (size=12151) 2024-11-15T00:46:22,209 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 626eba4d66894ec3b1277dd29ca5b71b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:22,209 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:22,209 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, priority=13, startTime=1731631582006; duration=0sec 2024-11-15T00:46:22,209 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:22,209 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:22,209 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:22,210 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:22,210 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C is initiating minor compaction (all files) 2024-11-15T00:46:22,210 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:22,211 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/242a3525ea4046cd8841614dfae8b065, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/272a61d92081406da4e0f59ffd4e1b84, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c894759d2c744551a9748a14323b40a0] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=35.8 K 2024-11-15T00:46:22,211 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 242a3525ea4046cd8841614dfae8b065, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731631577248 2024-11-15T00:46:22,211 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 272a61d92081406da4e0f59ffd4e1b84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631578537 2024-11-15T00:46:22,212 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c894759d2c744551a9748a14323b40a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631580480 2024-11-15T00:46:22,221 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:22,222 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c567918083754c43a645a0572ce2e712 is 50, key is test_row_0/C:col10/1731631580482/Put/seqid=0 2024-11-15T00:46:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742188_1364 (size=12459) 2024-11-15T00:46:22,261 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c567918083754c43a645a0572ce2e712 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c567918083754c43a645a0572ce2e712 2024-11-15T00:46:22,268 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into c567918083754c43a645a0572ce2e712(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:22,268 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:22,268 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, priority=13, startTime=1731631582006; duration=0sec 2024-11-15T00:46:22,268 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:22,268 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:22,505 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/f6907340a33746f1bdbe2734e38fe3a3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f6907340a33746f1bdbe2734e38fe3a3 2024-11-15T00:46:22,510 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into f6907340a33746f1bdbe2734e38fe3a3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:22,510 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:22,510 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, priority=13, startTime=1731631582006; duration=0sec 2024-11-15T00:46:22,510 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:22,510 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:22,605 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/3c5e38c823c2401da30f22bf41a7a160 2024-11-15T00:46:22,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/102f85389af04fd8bded5cfeaf25265c is 50, key is test_row_0/C:col10/1731631580565/Put/seqid=0 2024-11-15T00:46:22,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742189_1365 (size=12151) 2024-11-15T00:46:22,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:22,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631642809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631642809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:22,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631642809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:22,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631642859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:22,868 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in ccd4c1fbc17fcc85146b8fbdf8bdc2ff of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T00:46:22,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631642929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:22,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631642929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:22,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631642930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:23,091 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/102f85389af04fd8bded5cfeaf25265c 2024-11-15T00:46:23,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/a097cdf9ee70446fb460d5f6645ebf0c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a097cdf9ee70446fb460d5f6645ebf0c 2024-11-15T00:46:23,112 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a097cdf9ee70446fb460d5f6645ebf0c, entries=150, sequenceid=201, filesize=30.4 K 2024-11-15T00:46:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/3c5e38c823c2401da30f22bf41a7a160 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3c5e38c823c2401da30f22bf41a7a160 2024-11-15T00:46:23,127 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3c5e38c823c2401da30f22bf41a7a160, entries=150, sequenceid=201, filesize=11.9 K 2024-11-15T00:46:23,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/102f85389af04fd8bded5cfeaf25265c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/102f85389af04fd8bded5cfeaf25265c 2024-11-15T00:46:23,138 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/102f85389af04fd8bded5cfeaf25265c, entries=150, sequenceid=201, filesize=11.9 K 2024-11-15T00:46:23,147 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 1123ms, sequenceid=201, compaction requested=false 2024-11-15T00:46:23,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:23,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:23,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-15T00:46:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-15T00:46:23,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-15T00:46:23,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3850 sec 2024-11-15T00:46:23,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.3930 sec 2024-11-15T00:46:23,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T00:46:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:23,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:23,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:23,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:23,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:23,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:23,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:23,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150f5d39b0321943af88136d2734efde99_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631582806/Put/seqid=0 2024-11-15T00:46:23,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742190_1366 (size=14794) 2024-11-15T00:46:23,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631643259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:23,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:23,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631643262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:23,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631643297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:23,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:23,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631643399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:23,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631643410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:23,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:23,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631643429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:23,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631643621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:23,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631643633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:23,676 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:23,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631643668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:23,719 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150f5d39b0321943af88136d2734efde99_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150f5d39b0321943af88136d2734efde99_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:23,720 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/6fda998aac78446e94bc68d4d160ea37, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:23,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/6fda998aac78446e94bc68d4d160ea37 is 175, key is test_row_0/A:col10/1731631582806/Put/seqid=0 2024-11-15T00:46:23,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742191_1367 (size=39749) 2024-11-15T00:46:23,754 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/6fda998aac78446e94bc68d4d160ea37 2024-11-15T00:46:23,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/66baac4a4c914adb922dea6852138c82 is 50, key is test_row_0/B:col10/1731631582806/Put/seqid=0 2024-11-15T00:46:23,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742192_1368 (size=12151) 2024-11-15T00:46:23,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/66baac4a4c914adb922dea6852138c82 2024-11-15T00:46:23,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/e99827d472094f169f953d33c41af478 is 50, key is test_row_0/C:col10/1731631582806/Put/seqid=0 2024-11-15T00:46:23,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742193_1369 (size=12151) 2024-11-15T00:46:23,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/e99827d472094f169f953d33c41af478 2024-11-15T00:46:23,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/6fda998aac78446e94bc68d4d160ea37 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/6fda998aac78446e94bc68d4d160ea37 2024-11-15T00:46:23,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/6fda998aac78446e94bc68d4d160ea37, entries=200, sequenceid=221, filesize=38.8 K 2024-11-15T00:46:23,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/66baac4a4c914adb922dea6852138c82 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/66baac4a4c914adb922dea6852138c82 2024-11-15T00:46:23,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/66baac4a4c914adb922dea6852138c82, entries=150, sequenceid=221, filesize=11.9 K 2024-11-15T00:46:23,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/e99827d472094f169f953d33c41af478 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e99827d472094f169f953d33c41af478 2024-11-15T00:46:23,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e99827d472094f169f953d33c41af478, entries=150, sequenceid=221, filesize=11.9 K 2024-11-15T00:46:23,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 777ms, sequenceid=221, compaction requested=true 2024-11-15T00:46:23,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:23,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:23,933 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:23,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:23,933 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:23,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:23,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:23,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:23,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:23,935 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:23,935 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:23,935 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A is initiating minor compaction (all files) 2024-11-15T00:46:23,935 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B is initiating minor compaction (all files) 2024-11-15T00:46:23,935 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:23,935 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/626eba4d66894ec3b1277dd29ca5b71b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a097cdf9ee70446fb460d5f6645ebf0c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/6fda998aac78446e94bc68d4d160ea37] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=99.9 K 2024-11-15T00:46:23,935 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:23,935 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/626eba4d66894ec3b1277dd29ca5b71b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a097cdf9ee70446fb460d5f6645ebf0c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/6fda998aac78446e94bc68d4d160ea37] 2024-11-15T00:46:23,935 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:23,935 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f6907340a33746f1bdbe2734e38fe3a3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3c5e38c823c2401da30f22bf41a7a160, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/66baac4a4c914adb922dea6852138c82] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=35.9 K 2024-11-15T00:46:23,936 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f6907340a33746f1bdbe2734e38fe3a3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631580480 2024-11-15T00:46:23,936 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 626eba4d66894ec3b1277dd29ca5b71b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631580480 2024-11-15T00:46:23,936 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c5e38c823c2401da30f22bf41a7a160, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731631580528 2024-11-15T00:46:23,936 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting a097cdf9ee70446fb460d5f6645ebf0c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731631580528 2024-11-15T00:46:23,937 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 66baac4a4c914adb922dea6852138c82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731631582806 2024-11-15T00:46:23,937 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fda998aac78446e94bc68d4d160ea37, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731631582805 2024-11-15T00:46:23,944 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:23,949 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111571e22e874a644d7f8951aaacc35838e1_ccd4c1fbc17fcc85146b8fbdf8bdc2ff store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:23,949 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:23,950 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/7d582d9252cb4a1d8bad2184f6c1740c is 50, key is test_row_0/B:col10/1731631582806/Put/seqid=0 2024-11-15T00:46:23,951 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111571e22e874a644d7f8951aaacc35838e1_ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:23,951 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111571e22e874a644d7f8951aaacc35838e1_ccd4c1fbc17fcc85146b8fbdf8bdc2ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:23,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742194_1370 (size=12561) 2024-11-15T00:46:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:23,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T00:46:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:23,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742195_1371 (size=4469) 2024-11-15T00:46:23,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111575b034cecd274b4eac400ef4970874f1_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631583950/Put/seqid=0 2024-11-15T00:46:23,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742196_1372 (size=14794) 2024-11-15T00:46:23,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:23,984 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111575b034cecd274b4eac400ef4970874f1_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111575b034cecd274b4eac400ef4970874f1_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:23,985 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c04b7ec8601844e1b66fab3e3ff25613, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:23,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c04b7ec8601844e1b66fab3e3ff25613 is 175, key is test_row_0/A:col10/1731631583950/Put/seqid=0 2024-11-15T00:46:24,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742197_1373 (size=39749) 2024-11-15T00:46:24,005 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c04b7ec8601844e1b66fab3e3ff25613 2024-11-15T00:46:24,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/e1a84f0bd6f644ac9430996f95e92c81 is 50, key is test_row_0/B:col10/1731631583950/Put/seqid=0 2024-11-15T00:46:24,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742198_1374 (size=12151) 2024-11-15T00:46:24,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/e1a84f0bd6f644ac9430996f95e92c81 2024-11-15T00:46:24,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631644030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631644038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/d9648597b52b45d098bed45f477b23d3 is 50, key is test_row_0/C:col10/1731631583950/Put/seqid=0 2024-11-15T00:46:24,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631644040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742199_1375 (size=12151) 2024-11-15T00:46:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631644148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631644149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631644159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,361 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#A#compaction#311 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:24,362 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/b00742421f0c4aee9cece17f060a6997 is 175, key is test_row_0/A:col10/1731631582806/Put/seqid=0 2024-11-15T00:46:24,381 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/7d582d9252cb4a1d8bad2184f6c1740c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/7d582d9252cb4a1d8bad2184f6c1740c 2024-11-15T00:46:24,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631644372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631644372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631644381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742200_1376 (size=31515) 2024-11-15T00:46:24,401 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 7d582d9252cb4a1d8bad2184f6c1740c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:24,401 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:24,401 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, priority=13, startTime=1731631583933; duration=0sec 2024-11-15T00:46:24,401 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:24,401 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:24,401 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:24,402 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:24,402 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C is initiating minor compaction (all files) 2024-11-15T00:46:24,402 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:24,402 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c567918083754c43a645a0572ce2e712, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/102f85389af04fd8bded5cfeaf25265c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e99827d472094f169f953d33c41af478] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=35.9 K 2024-11-15T00:46:24,403 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c567918083754c43a645a0572ce2e712, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731631580480 2024-11-15T00:46:24,404 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 102f85389af04fd8bded5cfeaf25265c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731631580528 2024-11-15T00:46:24,404 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e99827d472094f169f953d33c41af478, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731631582806 2024-11-15T00:46:24,405 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/b00742421f0c4aee9cece17f060a6997 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b00742421f0c4aee9cece17f060a6997 2024-11-15T00:46:24,416 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into b00742421f0c4aee9cece17f060a6997(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:24,416 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:24,416 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, priority=13, startTime=1731631583933; duration=0sec 2024-11-15T00:46:24,416 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:24,416 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:24,423 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#C#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:24,423 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/b40953f5d3e54528adb2d05c5047d3b9 is 50, key is test_row_0/C:col10/1731631582806/Put/seqid=0 2024-11-15T00:46:24,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742201_1377 (size=12561) 2024-11-15T00:46:24,445 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/b40953f5d3e54528adb2d05c5047d3b9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/b40953f5d3e54528adb2d05c5047d3b9 2024-11-15T00:46:24,455 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into b40953f5d3e54528adb2d05c5047d3b9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:24,455 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:24,455 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, priority=13, startTime=1731631583934; duration=0sec 2024-11-15T00:46:24,455 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:24,455 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:24,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/d9648597b52b45d098bed45f477b23d3 2024-11-15T00:46:24,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c04b7ec8601844e1b66fab3e3ff25613 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c04b7ec8601844e1b66fab3e3ff25613 2024-11-15T00:46:24,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c04b7ec8601844e1b66fab3e3ff25613, entries=200, sequenceid=237, filesize=38.8 K 2024-11-15T00:46:24,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/e1a84f0bd6f644ac9430996f95e92c81 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/e1a84f0bd6f644ac9430996f95e92c81 2024-11-15T00:46:24,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/e1a84f0bd6f644ac9430996f95e92c81, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T00:46:24,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/d9648597b52b45d098bed45f477b23d3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d9648597b52b45d098bed45f477b23d3 2024-11-15T00:46:24,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d9648597b52b45d098bed45f477b23d3, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T00:46:24,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 554ms, sequenceid=237, compaction requested=false 2024-11-15T00:46:24,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:24,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T00:46:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:24,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:24,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:24,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:24,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115125e50e71ec04d45bd3019334b9bbb43_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631584697/Put/seqid=0 2024-11-15T00:46:24,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742202_1378 (size=12404) 2024-11-15T00:46:24,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:24,762 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115125e50e71ec04d45bd3019334b9bbb43_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115125e50e71ec04d45bd3019334b9bbb43_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:24,764 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8d1cbae21d744ad28947b94c7cc08a00, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:24,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8d1cbae21d744ad28947b94c7cc08a00 is 175, key is test_row_0/A:col10/1731631584697/Put/seqid=0 2024-11-15T00:46:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631644782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631644782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742203_1379 (size=31205) 2024-11-15T00:46:24,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631644796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631644870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,880 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in ccd4c1fbc17fcc85146b8fbdf8bdc2ff of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631644899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631644899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T00:46:24,925 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:24,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:24,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631644920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-15T00:46:24,953 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:46:24,954 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:24,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:46:25,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T00:46:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631645119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631645119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:25,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631645139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:25,199 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8d1cbae21d744ad28947b94c7cc08a00 2024-11-15T00:46:25,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/6cdfe4c500d646829db42a3c20a75bbb is 50, key is test_row_0/B:col10/1731631584697/Put/seqid=0 2024-11-15T00:46:25,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742204_1380 (size=12251) 2024-11-15T00:46:25,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T00:46:25,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:25,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:46:25,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T00:46:25,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:25,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631645442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:25,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631645443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:25,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:25,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631645460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:46:25,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T00:46:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:25,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/6cdfe4c500d646829db42a3c20a75bbb 2024-11-15T00:46:25,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/04fe84e67e9b48e48be62035dd976fbd is 50, key is test_row_0/C:col10/1731631584697/Put/seqid=0 2024-11-15T00:46:25,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742205_1381 (size=12251) 2024-11-15T00:46:25,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/04fe84e67e9b48e48be62035dd976fbd 2024-11-15T00:46:25,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8d1cbae21d744ad28947b94c7cc08a00 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d1cbae21d744ad28947b94c7cc08a00 2024-11-15T00:46:25,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d1cbae21d744ad28947b94c7cc08a00, entries=150, sequenceid=258, filesize=30.5 K 2024-11-15T00:46:25,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/6cdfe4c500d646829db42a3c20a75bbb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6cdfe4c500d646829db42a3c20a75bbb 2024-11-15T00:46:25,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6cdfe4c500d646829db42a3c20a75bbb, entries=150, sequenceid=258, filesize=12.0 K 2024-11-15T00:46:25,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/04fe84e67e9b48e48be62035dd976fbd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/04fe84e67e9b48e48be62035dd976fbd 2024-11-15T00:46:25,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/04fe84e67e9b48e48be62035dd976fbd, entries=150, sequenceid=258, filesize=12.0 K 2024-11-15T00:46:25,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 1033ms, sequenceid=258, compaction requested=true 2024-11-15T00:46:25,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:25,735 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:25,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:25,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:25,735 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:25,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:25,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:25,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:25,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:25,736 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:25,736 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A is initiating minor compaction (all files) 2024-11-15T00:46:25,736 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,736 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b00742421f0c4aee9cece17f060a6997, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c04b7ec8601844e1b66fab3e3ff25613, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d1cbae21d744ad28947b94c7cc08a00] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=100.1 K 2024-11-15T00:46:25,736 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,736 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b00742421f0c4aee9cece17f060a6997, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c04b7ec8601844e1b66fab3e3ff25613, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d1cbae21d744ad28947b94c7cc08a00] 2024-11-15T00:46:25,737 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b00742421f0c4aee9cece17f060a6997, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731631582806 2024-11-15T00:46:25,737 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36963 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:25,737 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B is initiating minor compaction (all files) 2024-11-15T00:46:25,737 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,737 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/7d582d9252cb4a1d8bad2184f6c1740c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/e1a84f0bd6f644ac9430996f95e92c81, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6cdfe4c500d646829db42a3c20a75bbb] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=36.1 K 2024-11-15T00:46:25,738 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting c04b7ec8601844e1b66fab3e3ff25613, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731631583207 2024-11-15T00:46:25,738 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d582d9252cb4a1d8bad2184f6c1740c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731631582806 2024-11-15T00:46:25,738 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d1cbae21d744ad28947b94c7cc08a00, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731631584030 2024-11-15T00:46:25,738 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e1a84f0bd6f644ac9430996f95e92c81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731631583207 2024-11-15T00:46:25,739 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cdfe4c500d646829db42a3c20a75bbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731631584030 2024-11-15T00:46:25,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T00:46:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,742 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T00:46:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:25,766 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:25,767 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/dcf9fa3c204b43aabeb173e21ecb1005 is 50, key is test_row_0/B:col10/1731631584697/Put/seqid=0 2024-11-15T00:46:25,779 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:25,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d7b7ea107f5b4ce49e2400d228b9add7_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631584772/Put/seqid=0 2024-11-15T00:46:25,812 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411153dc1cb705d3e4a81b603c0243306c70f_ccd4c1fbc17fcc85146b8fbdf8bdc2ff store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:25,818 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411153dc1cb705d3e4a81b603c0243306c70f_ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:25,818 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153dc1cb705d3e4a81b603c0243306c70f_ccd4c1fbc17fcc85146b8fbdf8bdc2ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:25,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742206_1382 (size=12763) 2024-11-15T00:46:25,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742207_1383 (size=12454) 2024-11-15T00:46:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:25,862 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/dcf9fa3c204b43aabeb173e21ecb1005 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/dcf9fa3c204b43aabeb173e21ecb1005 2024-11-15T00:46:25,864 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d7b7ea107f5b4ce49e2400d228b9add7_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d7b7ea107f5b4ce49e2400d228b9add7_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:25,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/e8635d6b77a148fbb81b01bbd7e3cd2f, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:25,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/e8635d6b77a148fbb81b01bbd7e3cd2f is 175, key is test_row_0/A:col10/1731631584772/Put/seqid=0 2024-11-15T00:46:25,874 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into dcf9fa3c204b43aabeb173e21ecb1005(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:25,874 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:25,874 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, priority=13, startTime=1731631585735; duration=0sec 2024-11-15T00:46:25,874 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:25,874 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:25,874 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:25,875 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36963 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:25,875 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C is initiating minor compaction (all files) 2024-11-15T00:46:25,875 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:25,876 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/b40953f5d3e54528adb2d05c5047d3b9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d9648597b52b45d098bed45f477b23d3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/04fe84e67e9b48e48be62035dd976fbd] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=36.1 K 2024-11-15T00:46:25,876 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting b40953f5d3e54528adb2d05c5047d3b9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731631582806 2024-11-15T00:46:25,876 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting d9648597b52b45d098bed45f477b23d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731631583207 2024-11-15T00:46:25,877 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 04fe84e67e9b48e48be62035dd976fbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731631584030 2024-11-15T00:46:25,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742208_1384 (size=4469) 2024-11-15T00:46:25,917 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:25,917 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c5c137f0fe2f453c94d9cd74fc8212dd is 50, key is test_row_0/C:col10/1731631584697/Put/seqid=0 2024-11-15T00:46:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742209_1385 (size=31255) 2024-11-15T00:46:25,963 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/e8635d6b77a148fbb81b01bbd7e3cd2f 2024-11-15T00:46:25,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742210_1386 (size=12763) 2024-11-15T00:46:25,983 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c5c137f0fe2f453c94d9cd74fc8212dd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c5c137f0fe2f453c94d9cd74fc8212dd 2024-11-15T00:46:25,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:25,993 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into c5c137f0fe2f453c94d9cd74fc8212dd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:25,993 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:25,993 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, priority=13, startTime=1731631585736; duration=0sec 2024-11-15T00:46:25,994 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:25,994 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:26,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/a2c8f7c6e9964990b81ace5a45f22c96 is 50, key is test_row_0/B:col10/1731631584772/Put/seqid=0 2024-11-15T00:46:26,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742211_1387 (size=12301) 2024-11-15T00:46:26,060 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/a2c8f7c6e9964990b81ace5a45f22c96 2024-11-15T00:46:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:46:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/49c524383d304a03ab1db0a90025c08f is 50, key is test_row_0/C:col10/1731631584772/Put/seqid=0 2024-11-15T00:46:26,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631646139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742212_1388 (size=12301) 2024-11-15T00:46:26,183 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/49c524383d304a03ab1db0a90025c08f 2024-11-15T00:46:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/e8635d6b77a148fbb81b01bbd7e3cd2f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e8635d6b77a148fbb81b01bbd7e3cd2f 2024-11-15T00:46:26,193 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e8635d6b77a148fbb81b01bbd7e3cd2f, entries=150, sequenceid=272, filesize=30.5 K 2024-11-15T00:46:26,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/a2c8f7c6e9964990b81ace5a45f22c96 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a2c8f7c6e9964990b81ace5a45f22c96 2024-11-15T00:46:26,200 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a2c8f7c6e9964990b81ace5a45f22c96, entries=150, sequenceid=272, filesize=12.0 K 2024-11-15T00:46:26,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/49c524383d304a03ab1db0a90025c08f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/49c524383d304a03ab1db0a90025c08f 2024-11-15T00:46:26,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631646166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631646171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,206 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/49c524383d304a03ab1db0a90025c08f, entries=150, sequenceid=272, filesize=12.0 K 2024-11-15T00:46:26,209 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 467ms, sequenceid=272, compaction requested=false 2024-11-15T00:46:26,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:26,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:26,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-15T00:46:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-15T00:46:26,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-15T00:46:26,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2570 sec 2024-11-15T00:46:26,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.2910 sec 2024-11-15T00:46:26,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T00:46:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:26,314 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#A#compaction#321 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:26,315 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/ce172e07a42e4b9c97710a02954453db is 175, key is test_row_0/A:col10/1731631584697/Put/seqid=0 2024-11-15T00:46:26,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d2d17e54ccc4ba893252e678cf111b2_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631586160/Put/seqid=0 2024-11-15T00:46:26,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742213_1389 (size=31717) 2024-11-15T00:46:26,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742214_1390 (size=14994) 2024-11-15T00:46:26,391 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/ce172e07a42e4b9c97710a02954453db as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/ce172e07a42e4b9c97710a02954453db 2024-11-15T00:46:26,396 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:26,409 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d2d17e54ccc4ba893252e678cf111b2_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d2d17e54ccc4ba893252e678cf111b2_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:26,411 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/e0400c2c27c74a0b841b9797fcd8e5b3, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:26,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/e0400c2c27c74a0b841b9797fcd8e5b3 is 175, key is test_row_0/A:col10/1731631586160/Put/seqid=0 2024-11-15T00:46:26,442 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into ce172e07a42e4b9c97710a02954453db(size=31.0 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:26,442 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:26,442 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, priority=13, startTime=1731631585735; duration=0sec 2024-11-15T00:46:26,442 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:26,442 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:26,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742215_1391 (size=39949) 2024-11-15T00:46:26,461 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/e0400c2c27c74a0b841b9797fcd8e5b3 2024-11-15T00:46:26,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/20d20505ae1e4f2a85f6f896746cd3fd is 50, key is test_row_0/B:col10/1731631586160/Put/seqid=0 2024-11-15T00:46:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631646445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631646449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631646449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742216_1392 (size=12301) 2024-11-15T00:46:26,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/20d20505ae1e4f2a85f6f896746cd3fd 2024-11-15T00:46:26,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/dff1c17032ce4508a2eaa7617f81e500 is 50, key is test_row_0/C:col10/1731631586160/Put/seqid=0 2024-11-15T00:46:26,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742217_1393 (size=12301) 2024-11-15T00:46:26,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/dff1c17032ce4508a2eaa7617f81e500 2024-11-15T00:46:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/e0400c2c27c74a0b841b9797fcd8e5b3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e0400c2c27c74a0b841b9797fcd8e5b3 2024-11-15T00:46:26,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e0400c2c27c74a0b841b9797fcd8e5b3, entries=200, sequenceid=293, filesize=39.0 K 2024-11-15T00:46:26,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/20d20505ae1e4f2a85f6f896746cd3fd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/20d20505ae1e4f2a85f6f896746cd3fd 2024-11-15T00:46:26,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/20d20505ae1e4f2a85f6f896746cd3fd, entries=150, sequenceid=293, filesize=12.0 K 2024-11-15T00:46:26,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/dff1c17032ce4508a2eaa7617f81e500 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/dff1c17032ce4508a2eaa7617f81e500 2024-11-15T00:46:26,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631646600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631646600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631646601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/dff1c17032ce4508a2eaa7617f81e500, entries=150, sequenceid=293, filesize=12.0 K 2024-11-15T00:46:26,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 313ms, sequenceid=293, compaction requested=true 2024-11-15T00:46:26,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:26,618 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:26,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:26,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:26,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:26,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:26,618 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:26,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:26,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:26,620 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102921 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:26,620 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A is initiating minor compaction (all files) 2024-11-15T00:46:26,620 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:26,621 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/ce172e07a42e4b9c97710a02954453db, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e8635d6b77a148fbb81b01bbd7e3cd2f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e0400c2c27c74a0b841b9797fcd8e5b3] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=100.5 K 2024-11-15T00:46:26,621 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:26,621 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/ce172e07a42e4b9c97710a02954453db, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e8635d6b77a148fbb81b01bbd7e3cd2f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e0400c2c27c74a0b841b9797fcd8e5b3] 2024-11-15T00:46:26,621 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:26,621 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B is initiating minor compaction (all files) 2024-11-15T00:46:26,621 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:26,621 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/dcf9fa3c204b43aabeb173e21ecb1005, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a2c8f7c6e9964990b81ace5a45f22c96, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/20d20505ae1e4f2a85f6f896746cd3fd] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=36.5 K 2024-11-15T00:46:26,621 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce172e07a42e4b9c97710a02954453db, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731631584030 2024-11-15T00:46:26,621 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting dcf9fa3c204b43aabeb173e21ecb1005, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731631584030 2024-11-15T00:46:26,622 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a2c8f7c6e9964990b81ace5a45f22c96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731631584767 2024-11-15T00:46:26,622 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 20d20505ae1e4f2a85f6f896746cd3fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731631586108 2024-11-15T00:46:26,624 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8635d6b77a148fbb81b01bbd7e3cd2f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731631584767 2024-11-15T00:46:26,625 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0400c2c27c74a0b841b9797fcd8e5b3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731631586108 2024-11-15T00:46:26,633 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#B#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:26,633 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/b41aa498e7f44beb90b075add5736caf is 50, key is test_row_0/B:col10/1731631586160/Put/seqid=0 2024-11-15T00:46:26,651 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:26,677 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411154dbdfc6a370d4569b43c1c0b4965cc81_ccd4c1fbc17fcc85146b8fbdf8bdc2ff store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:26,679 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411154dbdfc6a370d4569b43c1c0b4965cc81_ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:26,679 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154dbdfc6a370d4569b43c1c0b4965cc81_ccd4c1fbc17fcc85146b8fbdf8bdc2ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:26,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742218_1394 (size=12915) 2024-11-15T00:46:26,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742219_1395 (size=4469) 2024-11-15T00:46:26,725 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/b41aa498e7f44beb90b075add5736caf as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/b41aa498e7f44beb90b075add5736caf 2024-11-15T00:46:26,726 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#A#compaction#330 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:26,727 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c5775bbb9b504638acc11e015a0e45a4 is 175, key is test_row_0/A:col10/1731631586160/Put/seqid=0 2024-11-15T00:46:26,739 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into b41aa498e7f44beb90b075add5736caf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:26,739 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:26,739 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, priority=13, startTime=1731631586618; duration=0sec 2024-11-15T00:46:26,739 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:26,739 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:26,740 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:26,740 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:26,741 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C is initiating minor compaction (all files) 2024-11-15T00:46:26,741 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:26,741 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c5c137f0fe2f453c94d9cd74fc8212dd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/49c524383d304a03ab1db0a90025c08f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/dff1c17032ce4508a2eaa7617f81e500] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=36.5 K 2024-11-15T00:46:26,741 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c5c137f0fe2f453c94d9cd74fc8212dd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731631584030 2024-11-15T00:46:26,741 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 49c524383d304a03ab1db0a90025c08f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731631584767 2024-11-15T00:46:26,742 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting dff1c17032ce4508a2eaa7617f81e500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731631586108 2024-11-15T00:46:26,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742220_1396 (size=31869) 2024-11-15T00:46:26,764 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#C#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:26,765 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/ef8174b20d324602b42f47e8ddd8bdf8 is 50, key is test_row_0/C:col10/1731631586160/Put/seqid=0 2024-11-15T00:46:26,777 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c5775bbb9b504638acc11e015a0e45a4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c5775bbb9b504638acc11e015a0e45a4 2024-11-15T00:46:26,788 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into c5775bbb9b504638acc11e015a0e45a4(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:26,788 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:26,788 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, priority=13, startTime=1731631586618; duration=0sec 2024-11-15T00:46:26,788 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:26,788 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:26,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742221_1397 (size=12915) 2024-11-15T00:46:26,811 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/ef8174b20d324602b42f47e8ddd8bdf8 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ef8174b20d324602b42f47e8ddd8bdf8 2024-11-15T00:46:26,819 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into ef8174b20d324602b42f47e8ddd8bdf8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:26,819 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:26,819 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, priority=13, startTime=1731631586618; duration=0sec 2024-11-15T00:46:26,820 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:26,820 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:26,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T00:46:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:26,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115dc580d2cec194a73abc6c433f3003f7f_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631586363/Put/seqid=0 2024-11-15T00:46:26,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742222_1398 (size=12454) 2024-11-15T00:46:26,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60190 deadline: 1731631646900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,916 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in ccd4c1fbc17fcc85146b8fbdf8bdc2ff of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8312 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:26,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631646905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631646910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:26,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631646915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631647032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631647039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631647039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:46:27,099 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:27,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-15T00:46:27,103 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:27,103 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:27,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:46:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:46:27,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T00:46:27,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:27,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:27,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:27,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:27,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:27,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631647251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631647260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631647261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,284 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:27,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115dc580d2cec194a73abc6c433f3003f7f_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dc580d2cec194a73abc6c433f3003f7f_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:27,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8d75ad3601dd401a863aefe212cbdd59, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:27,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8d75ad3601dd401a863aefe212cbdd59 is 175, key is test_row_0/A:col10/1731631586363/Put/seqid=0 2024-11-15T00:46:27,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742223_1399 (size=31255) 2024-11-15T00:46:27,334 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8d75ad3601dd401a863aefe212cbdd59 2024-11-15T00:46:27,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/1745d816c43b42f0aa13586710429b12 is 50, key is test_row_0/B:col10/1731631586363/Put/seqid=0 2024-11-15T00:46:27,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742224_1400 (size=12301) 2024-11-15T00:46:27,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/1745d816c43b42f0aa13586710429b12 2024-11-15T00:46:27,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/e2693e0e26e742ab95ff302f4c6e1559 is 50, key is test_row_0/C:col10/1731631586363/Put/seqid=0 2024-11-15T00:46:27,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742225_1401 (size=12301) 2024-11-15T00:46:27,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/e2693e0e26e742ab95ff302f4c6e1559 2024-11-15T00:46:27,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8d75ad3601dd401a863aefe212cbdd59 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d75ad3601dd401a863aefe212cbdd59 2024-11-15T00:46:27,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d75ad3601dd401a863aefe212cbdd59, entries=150, sequenceid=311, filesize=30.5 K 2024-11-15T00:46:27,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/1745d816c43b42f0aa13586710429b12 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/1745d816c43b42f0aa13586710429b12 2024-11-15T00:46:27,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/1745d816c43b42f0aa13586710429b12, entries=150, sequenceid=311, filesize=12.0 K 2024-11-15T00:46:27,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/e2693e0e26e742ab95ff302f4c6e1559 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e2693e0e26e742ab95ff302f4c6e1559 2024-11-15T00:46:27,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e2693e0e26e742ab95ff302f4c6e1559, entries=150, sequenceid=311, filesize=12.0 K 2024-11-15T00:46:27,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 588ms, sequenceid=311, compaction requested=false 2024-11-15T00:46:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:46:27,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:27,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T00:46:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:27,421 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T00:46:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:27,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ba600182a12b4e5d9ed71c0161107410_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631586895/Put/seqid=0 2024-11-15T00:46:27,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742226_1402 (size=12454) 2024-11-15T00:46:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:27,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:27,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631647611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631647612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631647613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:46:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631647730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631647730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631647731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:27,867 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ba600182a12b4e5d9ed71c0161107410_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ba600182a12b4e5d9ed71c0161107410_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/d19102a48adb407995c82cf69843a93e, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/d19102a48adb407995c82cf69843a93e is 175, key is test_row_0/A:col10/1731631586895/Put/seqid=0 2024-11-15T00:46:27,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742227_1403 (size=31255) 2024-11-15T00:46:27,894 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/d19102a48adb407995c82cf69843a93e 2024-11-15T00:46:27,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631647939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631647939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/a85f79be6d0648b7ab2ed8e21af32569 is 50, key is test_row_0/B:col10/1731631586895/Put/seqid=0 2024-11-15T00:46:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631647949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:27,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742228_1404 (size=12301) 2024-11-15T00:46:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:46:28,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631648253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:28,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631648253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:28,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631648271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:28,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:28,362 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/a85f79be6d0648b7ab2ed8e21af32569 2024-11-15T00:46:28,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/d03d5fff32574eca90d0d6bcd4327c48 is 50, key is test_row_0/C:col10/1731631586895/Put/seqid=0 2024-11-15T00:46:28,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742229_1405 (size=12301) 2024-11-15T00:46:28,472 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/d03d5fff32574eca90d0d6bcd4327c48 2024-11-15T00:46:28,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/d19102a48adb407995c82cf69843a93e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/d19102a48adb407995c82cf69843a93e 2024-11-15T00:46:28,484 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/d19102a48adb407995c82cf69843a93e, entries=150, sequenceid=329, filesize=30.5 K 2024-11-15T00:46:28,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/a85f79be6d0648b7ab2ed8e21af32569 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a85f79be6d0648b7ab2ed8e21af32569 2024-11-15T00:46:28,494 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a85f79be6d0648b7ab2ed8e21af32569, entries=150, sequenceid=329, filesize=12.0 K 2024-11-15T00:46:28,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/d03d5fff32574eca90d0d6bcd4327c48 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d03d5fff32574eca90d0d6bcd4327c48 2024-11-15T00:46:28,518 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d03d5fff32574eca90d0d6bcd4327c48, entries=150, sequenceid=329, filesize=12.0 K 2024-11-15T00:46:28,519 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 1098ms, sequenceid=329, compaction requested=true 2024-11-15T00:46:28,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:28,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:28,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-15T00:46:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-15T00:46:28,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-15T00:46:28,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-11-15T00:46:28,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.4260 sec 2024-11-15T00:46:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:28,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T00:46:28,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:28,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:28,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:28,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:28,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:28,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:28,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111594c0d9711618424bb66bb60ff97603d9_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631588774/Put/seqid=0 2024-11-15T00:46:28,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742230_1406 (size=14994) 2024-11-15T00:46:28,808 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:28,813 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111594c0d9711618424bb66bb60ff97603d9_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111594c0d9711618424bb66bb60ff97603d9_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:28,814 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/28cdf7cac4e646d9b5824af99f9e81b7, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:28,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/28cdf7cac4e646d9b5824af99f9e81b7 is 175, key is test_row_0/A:col10/1731631588774/Put/seqid=0 2024-11-15T00:46:28,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742231_1407 (size=39949) 2024-11-15T00:46:28,822 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/28cdf7cac4e646d9b5824af99f9e81b7 2024-11-15T00:46:28,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/88a22a8dfaa84adcaf1f571f8814d97a is 50, key is test_row_0/B:col10/1731631588774/Put/seqid=0 2024-11-15T00:46:28,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631648834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:28,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:28,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631648840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:28,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742232_1408 (size=12301) 2024-11-15T00:46:28,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/88a22a8dfaa84adcaf1f571f8814d97a 2024-11-15T00:46:28,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631648849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:28,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:28,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:28,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/95d1883d73654fe9aaa60f43a3321434 is 50, key is test_row_0/C:col10/1731631588774/Put/seqid=0 2024-11-15T00:46:28,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742233_1409 (size=12301) 2024-11-15T00:46:28,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/95d1883d73654fe9aaa60f43a3321434 2024-11-15T00:46:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/28cdf7cac4e646d9b5824af99f9e81b7 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/28cdf7cac4e646d9b5824af99f9e81b7 2024-11-15T00:46:28,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/28cdf7cac4e646d9b5824af99f9e81b7, entries=200, sequenceid=344, filesize=39.0 K 2024-11-15T00:46:28,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/88a22a8dfaa84adcaf1f571f8814d97a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/88a22a8dfaa84adcaf1f571f8814d97a 2024-11-15T00:46:28,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/88a22a8dfaa84adcaf1f571f8814d97a, entries=150, sequenceid=344, filesize=12.0 K 2024-11-15T00:46:28,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/95d1883d73654fe9aaa60f43a3321434 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/95d1883d73654fe9aaa60f43a3321434 2024-11-15T00:46:28,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/95d1883d73654fe9aaa60f43a3321434, entries=150, sequenceid=344, filesize=12.0 K 2024-11-15T00:46:28,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 154ms, sequenceid=344, compaction requested=true 2024-11-15T00:46:28,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:28,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:28,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:28,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:28,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:28,931 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:28,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:28,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:46:28,932 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:28,932 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:28,932 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A is initiating minor compaction (all files) 2024-11-15T00:46:28,933 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:28,933 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c5775bbb9b504638acc11e015a0e45a4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d75ad3601dd401a863aefe212cbdd59, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/d19102a48adb407995c82cf69843a93e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/28cdf7cac4e646d9b5824af99f9e81b7] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=131.2 K 2024-11-15T00:46:28,933 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:28,933 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c5775bbb9b504638acc11e015a0e45a4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d75ad3601dd401a863aefe212cbdd59, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/d19102a48adb407995c82cf69843a93e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/28cdf7cac4e646d9b5824af99f9e81b7] 2024-11-15T00:46:28,933 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5775bbb9b504638acc11e015a0e45a4, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731631586108 2024-11-15T00:46:28,933 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:28,934 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B is initiating minor compaction (all files) 2024-11-15T00:46:28,934 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:28,934 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/b41aa498e7f44beb90b075add5736caf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/1745d816c43b42f0aa13586710429b12, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a85f79be6d0648b7ab2ed8e21af32569, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/88a22a8dfaa84adcaf1f571f8814d97a] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=48.7 K 2024-11-15T00:46:28,934 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting b41aa498e7f44beb90b075add5736caf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731631586108 2024-11-15T00:46:28,934 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 1745d816c43b42f0aa13586710429b12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731631586363 2024-11-15T00:46:28,934 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a85f79be6d0648b7ab2ed8e21af32569, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731631586895 2024-11-15T00:46:28,935 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 88a22a8dfaa84adcaf1f571f8814d97a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731631587602 2024-11-15T00:46:28,935 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d75ad3601dd401a863aefe212cbdd59, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731631586363 2024-11-15T00:46:28,935 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting d19102a48adb407995c82cf69843a93e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731631586895 2024-11-15T00:46:28,936 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28cdf7cac4e646d9b5824af99f9e81b7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731631587602 2024-11-15T00:46:28,946 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#B#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:28,946 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/9922684cf8454dc092487f2c28a03ec0 is 50, key is test_row_0/B:col10/1731631588774/Put/seqid=0 2024-11-15T00:46:28,947 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:28,953 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411156774c0d5918f49b2816d3d49c07853d6_ccd4c1fbc17fcc85146b8fbdf8bdc2ff store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:28,955 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411156774c0d5918f49b2816d3d49c07853d6_ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:28,956 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156774c0d5918f49b2816d3d49c07853d6_ccd4c1fbc17fcc85146b8fbdf8bdc2ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:28,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742234_1410 (size=13051) 2024-11-15T00:46:28,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742235_1411 (size=4469) 2024-11-15T00:46:28,964 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#A#compaction#342 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:28,965 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/9d71a2f39a1b40edaf7dc366c3161286 is 175, key is test_row_0/A:col10/1731631588774/Put/seqid=0 2024-11-15T00:46:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:28,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T00:46:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:28,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742236_1412 (size=32005) 2024-11-15T00:46:28,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158ead79a1b50f4ecc9d39fd7d7f902a71_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631588836/Put/seqid=0 2024-11-15T00:46:28,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742237_1413 (size=14994) 2024-11-15T00:46:29,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631649030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631649035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631649036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631649150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631649150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631649159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:46:29,250 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:29,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-15T00:46:29,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:46:29,255 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:29,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:46:29,364 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/9922684cf8454dc092487f2c28a03ec0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/9922684cf8454dc092487f2c28a03ec0 2024-11-15T00:46:29,370 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 9922684cf8454dc092487f2c28a03ec0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:29,371 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:29,371 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, priority=12, startTime=1731631588931; duration=0sec 2024-11-15T00:46:29,371 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:29,371 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:29,371 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:29,372 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:29,372 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C is initiating minor compaction (all files) 2024-11-15T00:46:29,372 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:29,372 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ef8174b20d324602b42f47e8ddd8bdf8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e2693e0e26e742ab95ff302f4c6e1559, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d03d5fff32574eca90d0d6bcd4327c48, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/95d1883d73654fe9aaa60f43a3321434] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=48.7 K 2024-11-15T00:46:29,373 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ef8174b20d324602b42f47e8ddd8bdf8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731631586108 2024-11-15T00:46:29,374 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e2693e0e26e742ab95ff302f4c6e1559, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731631586363 2024-11-15T00:46:29,374 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting d03d5fff32574eca90d0d6bcd4327c48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731631586895 2024-11-15T00:46:29,375 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/9d71a2f39a1b40edaf7dc366c3161286 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9d71a2f39a1b40edaf7dc366c3161286 2024-11-15T00:46:29,375 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d1883d73654fe9aaa60f43a3321434, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731631587602 2024-11-15T00:46:29,380 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 9d71a2f39a1b40edaf7dc366c3161286(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:29,380 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:29,381 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, priority=12, startTime=1731631588931; duration=0sec 2024-11-15T00:46:29,381 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:29,381 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:29,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631649370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631649370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631649384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,393 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:29,398 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158ead79a1b50f4ecc9d39fd7d7f902a71_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158ead79a1b50f4ecc9d39fd7d7f902a71_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:29,399 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/fd78430efa67459897fc5ca1338b88ce, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:29,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/fd78430efa67459897fc5ca1338b88ce is 175, key is test_row_0/A:col10/1731631588836/Put/seqid=0 2024-11-15T00:46:29,404 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#C#compaction#344 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:29,404 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/0c4a417df2f64fd6be25b5703020568d is 50, key is test_row_0/C:col10/1731631588774/Put/seqid=0 2024-11-15T00:46:29,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T00:46:29,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:29,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:29,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:29,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742238_1414 (size=39949) 2024-11-15T00:46:29,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742239_1415 (size=13051) 2024-11-15T00:46:29,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T00:46:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:29,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:46:29,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631649689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631649689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631649699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:29,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T00:46:29,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:29,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:29,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:29,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,825 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/fd78430efa67459897fc5ca1338b88ce 2024-11-15T00:46:29,835 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/0c4a417df2f64fd6be25b5703020568d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/0c4a417df2f64fd6be25b5703020568d 2024-11-15T00:46:29,839 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 0c4a417df2f64fd6be25b5703020568d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:29,839 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:29,839 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, priority=12, startTime=1731631588931; duration=0sec 2024-11-15T00:46:29,839 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:29,839 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:29,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/ec768df608b543e5a0c6a5323efe60eb is 50, key is test_row_0/B:col10/1731631588836/Put/seqid=0 2024-11-15T00:46:29,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742240_1416 (size=12301) 2024-11-15T00:46:29,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/ec768df608b543e5a0c6a5323efe60eb 2024-11-15T00:46:29,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T00:46:29,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:29,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:29,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:29,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:46:29,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/ddde1e21810947179c88970e0cbdf926 is 50, key is test_row_0/C:col10/1731631588836/Put/seqid=0 2024-11-15T00:46:29,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742241_1417 (size=12301) 2024-11-15T00:46:29,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/ddde1e21810947179c88970e0cbdf926 2024-11-15T00:46:29,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/fd78430efa67459897fc5ca1338b88ce as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/fd78430efa67459897fc5ca1338b88ce 2024-11-15T00:46:29,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/fd78430efa67459897fc5ca1338b88ce, entries=200, sequenceid=362, filesize=39.0 K 2024-11-15T00:46:29,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/ec768df608b543e5a0c6a5323efe60eb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ec768df608b543e5a0c6a5323efe60eb 2024-11-15T00:46:29,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ec768df608b543e5a0c6a5323efe60eb, entries=150, sequenceid=362, filesize=12.0 K 2024-11-15T00:46:29,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/ddde1e21810947179c88970e0cbdf926 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ddde1e21810947179c88970e0cbdf926 2024-11-15T00:46:29,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ddde1e21810947179c88970e0cbdf926, entries=150, sequenceid=362, filesize=12.0 K 2024-11-15T00:46:29,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 977ms, sequenceid=362, compaction requested=false 2024-11-15T00:46:29,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:30,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T00:46:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:30,025 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T00:46:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:30,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155144717cf9bc41f8a5dc3addf4ab6ff8_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631589009/Put/seqid=0 2024-11-15T00:46:30,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742242_1418 (size=12454) 2024-11-15T00:46:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:30,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631650289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631650291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631650292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,332 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:46:30,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:46:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631650409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631650419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631650419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:30,448 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155144717cf9bc41f8a5dc3addf4ab6ff8_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155144717cf9bc41f8a5dc3addf4ab6ff8_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:30,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c6b337c67af141aa9d88f039a63fbf7f, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:30,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c6b337c67af141aa9d88f039a63fbf7f is 175, key is test_row_0/A:col10/1731631589009/Put/seqid=0 2024-11-15T00:46:30,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742243_1419 (size=31255) 2024-11-15T00:46:30,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631650636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631650640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631650640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,880 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c6b337c67af141aa9d88f039a63fbf7f 2024-11-15T00:46:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/ac9dd99689854f9381d11805fd9759bb is 50, key is test_row_0/B:col10/1731631589009/Put/seqid=0 2024-11-15T00:46:30,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742244_1420 (size=12301) 2024-11-15T00:46:30,918 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/ac9dd99689854f9381d11805fd9759bb 2024-11-15T00:46:30,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c044da4ce3ad4399812609a0ab68d6e2 is 50, key is test_row_0/C:col10/1731631589009/Put/seqid=0 2024-11-15T00:46:30,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631650959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631650959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631650960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:30,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742245_1421 (size=12301) 2024-11-15T00:46:30,998 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c044da4ce3ad4399812609a0ab68d6e2 2024-11-15T00:46:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c6b337c67af141aa9d88f039a63fbf7f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c6b337c67af141aa9d88f039a63fbf7f 2024-11-15T00:46:31,008 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c6b337c67af141aa9d88f039a63fbf7f, entries=150, sequenceid=379, filesize=30.5 K 2024-11-15T00:46:31,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/ac9dd99689854f9381d11805fd9759bb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ac9dd99689854f9381d11805fd9759bb 2024-11-15T00:46:31,014 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ac9dd99689854f9381d11805fd9759bb, entries=150, sequenceid=379, filesize=12.0 K 2024-11-15T00:46:31,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c044da4ce3ad4399812609a0ab68d6e2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c044da4ce3ad4399812609a0ab68d6e2 2024-11-15T00:46:31,021 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c044da4ce3ad4399812609a0ab68d6e2, entries=150, sequenceid=379, filesize=12.0 K 2024-11-15T00:46:31,022 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 997ms, sequenceid=379, compaction requested=true 2024-11-15T00:46:31,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:31,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:31,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-15T00:46:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-15T00:46:31,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-15T00:46:31,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7680 sec 2024-11-15T00:46:31,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.7740 sec 2024-11-15T00:46:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:46:31,400 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:31,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-15T00:46:31,402 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:46:31,403 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:31,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:31,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T00:46:31,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:31,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:31,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:31,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:31,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:31,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:46:31,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155543e76b99bb412bb39ece21851848d3_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631591491/Put/seqid=0 2024-11-15T00:46:31,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:31,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:31,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:31,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:31,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:31,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:31,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631651540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:31,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631651554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:31,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742246_1422 (size=12454) 2024-11-15T00:46:31,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631651561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:31,582 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155543e76b99bb412bb39ece21851848d3_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155543e76b99bb412bb39ece21851848d3_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:31,583 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/f5cdd3c70ea84d1990b09a017371b70c, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:31,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/f5cdd3c70ea84d1990b09a017371b70c is 175, key is test_row_0/A:col10/1731631591491/Put/seqid=0 2024-11-15T00:46:31,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742247_1423 (size=31255) 2024-11-15T00:46:31,629 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/f5cdd3c70ea84d1990b09a017371b70c 2024-11-15T00:46:31,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/4171f314cc864fea9c6d63bb204b1e90 is 50, key is test_row_0/B:col10/1731631591491/Put/seqid=0 2024-11-15T00:46:31,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631651673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:31,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631651679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:31,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:31,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742248_1424 (size=12301) 2024-11-15T00:46:31,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:31,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:31,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:31,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:31,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:31,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:31,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631651692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:46:31,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:31,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:31,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:31,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:31,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:31,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:31,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:31,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631651900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:31,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631651900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631651921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:32,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:32,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:46:32,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/4171f314cc864fea9c6d63bb204b1e90 2024-11-15T00:46:32,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/72346911dc9a4b25b8f07e669f033963 is 50, key is test_row_0/C:col10/1731631591491/Put/seqid=0 2024-11-15T00:46:32,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742249_1425 (size=12301) 2024-11-15T00:46:32,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/72346911dc9a4b25b8f07e669f033963 2024-11-15T00:46:32,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/f5cdd3c70ea84d1990b09a017371b70c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f5cdd3c70ea84d1990b09a017371b70c 2024-11-15T00:46:32,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:32,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:32,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f5cdd3c70ea84d1990b09a017371b70c, entries=150, sequenceid=398, filesize=30.5 K 2024-11-15T00:46:32,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/4171f314cc864fea9c6d63bb204b1e90 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/4171f314cc864fea9c6d63bb204b1e90 2024-11-15T00:46:32,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/4171f314cc864fea9c6d63bb204b1e90, entries=150, sequenceid=398, filesize=12.0 K 2024-11-15T00:46:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/72346911dc9a4b25b8f07e669f033963 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/72346911dc9a4b25b8f07e669f033963 2024-11-15T00:46:32,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/72346911dc9a4b25b8f07e669f033963, entries=150, sequenceid=398, filesize=12.0 K 2024-11-15T00:46:32,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 711ms, sequenceid=398, compaction requested=true 2024-11-15T00:46:32,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:32,205 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:46:32,205 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:32,209 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:32,209 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:32,209 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B is initiating minor compaction (all files) 2024-11-15T00:46:32,209 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A is initiating minor compaction (all files) 2024-11-15T00:46:32,209 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,209 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,209 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/9922684cf8454dc092487f2c28a03ec0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ec768df608b543e5a0c6a5323efe60eb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ac9dd99689854f9381d11805fd9759bb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/4171f314cc864fea9c6d63bb204b1e90] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=48.8 K 2024-11-15T00:46:32,209 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9d71a2f39a1b40edaf7dc366c3161286, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/fd78430efa67459897fc5ca1338b88ce, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c6b337c67af141aa9d88f039a63fbf7f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f5cdd3c70ea84d1990b09a017371b70c] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=131.3 K 2024-11-15T00:46:32,209 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,209 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9d71a2f39a1b40edaf7dc366c3161286, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/fd78430efa67459897fc5ca1338b88ce, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c6b337c67af141aa9d88f039a63fbf7f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f5cdd3c70ea84d1990b09a017371b70c] 2024-11-15T00:46:32,211 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 9922684cf8454dc092487f2c28a03ec0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731631587602 2024-11-15T00:46:32,211 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d71a2f39a1b40edaf7dc366c3161286, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731631587602 2024-11-15T00:46:32,213 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ec768df608b543e5a0c6a5323efe60eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731631588832 2024-11-15T00:46:32,213 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd78430efa67459897fc5ca1338b88ce, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731631588832 2024-11-15T00:46:32,214 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ac9dd99689854f9381d11805fd9759bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731631589009 2024-11-15T00:46:32,214 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6b337c67af141aa9d88f039a63fbf7f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731631589009 2024-11-15T00:46:32,214 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5cdd3c70ea84d1990b09a017371b70c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731631590289 2024-11-15T00:46:32,214 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 4171f314cc864fea9c6d63bb204b1e90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731631590289 2024-11-15T00:46:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:32,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T00:46:32,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:32,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:32,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:32,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:32,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:32,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:32,237 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:32,251 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:32,251 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/dc953eafb2de4cc394948b11768148cd is 50, key is test_row_0/B:col10/1731631591491/Put/seqid=0 2024-11-15T00:46:32,272 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111565b4f3cca5d549b1937b4e583ec5095e_ccd4c1fbc17fcc85146b8fbdf8bdc2ff store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:32,275 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111565b4f3cca5d549b1937b4e583ec5095e_ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:32,275 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111565b4f3cca5d549b1937b4e583ec5095e_ccd4c1fbc17fcc85146b8fbdf8bdc2ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:32,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159b1bc2d87afe4ad1827d2ec7786253b4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631592222/Put/seqid=0 2024-11-15T00:46:32,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742252_1428 (size=14994) 2024-11-15T00:46:32,294 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:32,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742250_1426 (size=13187) 2024-11-15T00:46:32,299 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159b1bc2d87afe4ad1827d2ec7786253b4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b1bc2d87afe4ad1827d2ec7786253b4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:32,300 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c4347dba47404509abee595b8a3fc51b, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:32,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c4347dba47404509abee595b8a3fc51b is 175, key is test_row_0/A:col10/1731631592222/Put/seqid=0 2024-11-15T00:46:32,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742251_1427 (size=4469) 2024-11-15T00:46:32,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631652305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631652306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631652310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742253_1429 (size=39949) 2024-11-15T00:46:32,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631652432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631652433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631652436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:46:32,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:32,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:32,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631652660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631652660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631652663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,702 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/dc953eafb2de4cc394948b11768148cd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/dc953eafb2de4cc394948b11768148cd 2024-11-15T00:46:32,709 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#A#compaction#353 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:32,709 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/a33270521613499db099ca57dc8b10ac is 175, key is test_row_0/A:col10/1731631591491/Put/seqid=0 2024-11-15T00:46:32,714 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into dc953eafb2de4cc394948b11768148cd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:32,714 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:32,714 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, priority=12, startTime=1731631592205; duration=0sec 2024-11-15T00:46:32,714 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:32,714 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:32,714 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:32,715 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:32,715 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C is initiating minor compaction (all files) 2024-11-15T00:46:32,715 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,716 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/0c4a417df2f64fd6be25b5703020568d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ddde1e21810947179c88970e0cbdf926, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c044da4ce3ad4399812609a0ab68d6e2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/72346911dc9a4b25b8f07e669f033963] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=48.8 K 2024-11-15T00:46:32,716 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c4a417df2f64fd6be25b5703020568d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731631587602 2024-11-15T00:46:32,716 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ddde1e21810947179c88970e0cbdf926, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731631588832 2024-11-15T00:46:32,716 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c044da4ce3ad4399812609a0ab68d6e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731631589009 2024-11-15T00:46:32,717 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 72346911dc9a4b25b8f07e669f033963, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731631590289 2024-11-15T00:46:32,747 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:32,748 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/fccd52e1a72f42d4b93a07644bb635c3 is 50, key is test_row_0/C:col10/1731631591491/Put/seqid=0 2024-11-15T00:46:32,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742254_1430 (size=32141) 2024-11-15T00:46:32,760 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/a33270521613499db099ca57dc8b10ac as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a33270521613499db099ca57dc8b10ac 2024-11-15T00:46:32,766 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c4347dba47404509abee595b8a3fc51b 2024-11-15T00:46:32,768 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into a33270521613499db099ca57dc8b10ac(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:32,768 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:32,768 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, priority=12, startTime=1731631592204; duration=0sec 2024-11-15T00:46:32,768 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:32,768 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:32,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742255_1431 (size=13187) 2024-11-15T00:46:32,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:32,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:32,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/41b48ea4a6b843878667e0ce07a0ea27 is 50, key is test_row_0/B:col10/1731631592222/Put/seqid=0 2024-11-15T00:46:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742256_1432 (size=12301) 2024-11-15T00:46:32,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/41b48ea4a6b843878667e0ce07a0ea27 2024-11-15T00:46:32,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/ea9606f470d24e8db59b636fce3943d4 is 50, key is test_row_0/C:col10/1731631592222/Put/seqid=0 2024-11-15T00:46:32,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742257_1433 (size=12301) 2024-11-15T00:46:32,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/ea9606f470d24e8db59b636fce3943d4 2024-11-15T00:46:32,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c4347dba47404509abee595b8a3fc51b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c4347dba47404509abee595b8a3fc51b 2024-11-15T00:46:32,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c4347dba47404509abee595b8a3fc51b, entries=200, sequenceid=412, filesize=39.0 K 2024-11-15T00:46:32,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/41b48ea4a6b843878667e0ce07a0ea27 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/41b48ea4a6b843878667e0ce07a0ea27 2024-11-15T00:46:32,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:32,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:32,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:32,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:32,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/41b48ea4a6b843878667e0ce07a0ea27, entries=150, sequenceid=412, filesize=12.0 K 2024-11-15T00:46:32,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/ea9606f470d24e8db59b636fce3943d4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ea9606f470d24e8db59b636fce3943d4 2024-11-15T00:46:32,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631652979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:32,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631652979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:33,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ea9606f470d24e8db59b636fce3943d4, entries=150, sequenceid=412, filesize=12.0 K 2024-11-15T00:46:33,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 786ms, sequenceid=412, compaction requested=false 2024-11-15T00:46:33,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:33,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T00:46:33,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:33,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:33,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:33,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:33,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:33,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:33,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157ec8f48e033c449eb35a1651080b8b77_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631593009/Put/seqid=0 2024-11-15T00:46:33,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742258_1434 (size=14994) 2024-11-15T00:46:33,115 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:33,118 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157ec8f48e033c449eb35a1651080b8b77_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157ec8f48e033c449eb35a1651080b8b77_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:33,119 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/627147f950394d69906d82f1a6d7bf90, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:33,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/627147f950394d69906d82f1a6d7bf90 is 175, key is test_row_0/A:col10/1731631593009/Put/seqid=0 2024-11-15T00:46:33,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:33,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:33,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742259_1435 (size=39949) 2024-11-15T00:46:33,137 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/627147f950394d69906d82f1a6d7bf90 2024-11-15T00:46:33,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/0dc3846cd4cf49e09c2c148a3c1f751b is 50, key is test_row_0/B:col10/1731631593009/Put/seqid=0 2024-11-15T00:46:33,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742260_1436 (size=12301) 2024-11-15T00:46:33,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/0dc3846cd4cf49e09c2c148a3c1f751b 2024-11-15T00:46:33,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/643206632cd0452296af6fd5326f6a73 is 50, key is test_row_0/C:col10/1731631593009/Put/seqid=0 2024-11-15T00:46:33,197 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/fccd52e1a72f42d4b93a07644bb635c3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/fccd52e1a72f42d4b93a07644bb635c3 2024-11-15T00:46:33,202 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into fccd52e1a72f42d4b93a07644bb635c3(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:33,202 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:33,202 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, priority=12, startTime=1731631592205; duration=0sec 2024-11-15T00:46:33,202 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:33,202 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:33,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742261_1437 (size=12301) 2024-11-15T00:46:33,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631653218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:33,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:33,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631653339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:33,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:33,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:33,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:33,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631653499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:33,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631653499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:46:33,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631653560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:33,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:33,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/643206632cd0452296af6fd5326f6a73 2024-11-15T00:46:33,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/627147f950394d69906d82f1a6d7bf90 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/627147f950394d69906d82f1a6d7bf90 2024-11-15T00:46:33,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/627147f950394d69906d82f1a6d7bf90, entries=200, sequenceid=433, filesize=39.0 K 2024-11-15T00:46:33,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/0dc3846cd4cf49e09c2c148a3c1f751b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/0dc3846cd4cf49e09c2c148a3c1f751b 2024-11-15T00:46:33,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/0dc3846cd4cf49e09c2c148a3c1f751b, entries=150, sequenceid=433, filesize=12.0 K 2024-11-15T00:46:33,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/643206632cd0452296af6fd5326f6a73 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/643206632cd0452296af6fd5326f6a73 2024-11-15T00:46:33,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/643206632cd0452296af6fd5326f6a73, entries=150, sequenceid=433, filesize=12.0 K 2024-11-15T00:46:33,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 621ms, sequenceid=433, compaction requested=true 2024-11-15T00:46:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:33,635 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:33,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:33,635 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:33,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:33,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:33,636 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:33,636 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:33,636 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A is initiating minor compaction (all files) 2024-11-15T00:46:33,636 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B is initiating minor compaction (all files) 2024-11-15T00:46:33,636 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,636 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,636 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a33270521613499db099ca57dc8b10ac, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c4347dba47404509abee595b8a3fc51b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/627147f950394d69906d82f1a6d7bf90] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=109.4 K 2024-11-15T00:46:33,636 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/dc953eafb2de4cc394948b11768148cd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/41b48ea4a6b843878667e0ce07a0ea27, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/0dc3846cd4cf49e09c2c148a3c1f751b] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=36.9 K 2024-11-15T00:46:33,636 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,636 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a33270521613499db099ca57dc8b10ac, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c4347dba47404509abee595b8a3fc51b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/627147f950394d69906d82f1a6d7bf90] 2024-11-15T00:46:33,637 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting dc953eafb2de4cc394948b11768148cd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731631590289 2024-11-15T00:46:33,637 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting a33270521613499db099ca57dc8b10ac, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731631590289 2024-11-15T00:46:33,637 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b48ea4a6b843878667e0ce07a0ea27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731631591533 2024-11-15T00:46:33,637 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4347dba47404509abee595b8a3fc51b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731631591533 2024-11-15T00:46:33,637 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc3846cd4cf49e09c2c148a3c1f751b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731631592305 2024-11-15T00:46:33,637 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 627147f950394d69906d82f1a6d7bf90, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731631592303 2024-11-15T00:46:33,642 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:33,643 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115c60a3926219649f4827540c892b651c2_ccd4c1fbc17fcc85146b8fbdf8bdc2ff store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:33,644 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:33,645 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115c60a3926219649f4827540c892b651c2_ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:33,645 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/6f8068f8d0f4466fb48fa0122dcec75c is 50, key is test_row_0/B:col10/1731631593009/Put/seqid=0 2024-11-15T00:46:33,645 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c60a3926219649f4827540c892b651c2_ccd4c1fbc17fcc85146b8fbdf8bdc2ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:33,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742263_1439 (size=4469) 2024-11-15T00:46:33,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742262_1438 (size=13289) 2024-11-15T00:46:33,657 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#A#compaction#362 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:33,657 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/b8ae7141e7bf4319803d01eb8b80d24d is 175, key is test_row_0/A:col10/1731631593009/Put/seqid=0 2024-11-15T00:46:33,660 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/6f8068f8d0f4466fb48fa0122dcec75c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6f8068f8d0f4466fb48fa0122dcec75c 2024-11-15T00:46:33,664 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 6f8068f8d0f4466fb48fa0122dcec75c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:33,664 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:33,664 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, priority=13, startTime=1731631593635; duration=0sec 2024-11-15T00:46:33,664 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:33,664 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:33,664 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:33,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742264_1440 (size=32243) 2024-11-15T00:46:33,665 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:33,665 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C is initiating minor compaction (all files) 2024-11-15T00:46:33,665 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,666 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/fccd52e1a72f42d4b93a07644bb635c3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ea9606f470d24e8db59b636fce3943d4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/643206632cd0452296af6fd5326f6a73] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=36.9 K 2024-11-15T00:46:33,666 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting fccd52e1a72f42d4b93a07644bb635c3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731631590289 2024-11-15T00:46:33,666 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ea9606f470d24e8db59b636fce3943d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731631591533 2024-11-15T00:46:33,667 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 643206632cd0452296af6fd5326f6a73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731631592305 2024-11-15T00:46:33,676 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#C#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:33,677 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/28b7660d5e5c4ce8a4158c62e3ed467d is 50, key is test_row_0/C:col10/1731631593009/Put/seqid=0 2024-11-15T00:46:33,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742265_1441 (size=13289) 2024-11-15T00:46:33,686 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/28b7660d5e5c4ce8a4158c62e3ed467d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/28b7660d5e5c4ce8a4158c62e3ed467d 2024-11-15T00:46:33,692 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into 28b7660d5e5c4ce8a4158c62e3ed467d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:33,692 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:33,693 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, priority=13, startTime=1731631593635; duration=0sec 2024-11-15T00:46:33,693 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:33,693 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:33,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:46:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:33,733 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:46:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:33,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115409127f696c445a29fb8ca1dc9525b44_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631593107/Put/seqid=0 2024-11-15T00:46:33,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742266_1442 (size=12454) 2024-11-15T00:46:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. as already flushing 2024-11-15T00:46:34,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631654043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:34,071 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/b8ae7141e7bf4319803d01eb8b80d24d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b8ae7141e7bf4319803d01eb8b80d24d 2024-11-15T00:46:34,090 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into b8ae7141e7bf4319803d01eb8b80d24d(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:34,090 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:34,090 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, priority=13, startTime=1731631593635; duration=0sec 2024-11-15T00:46:34,090 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:34,090 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:34,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:34,150 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115409127f696c445a29fb8ca1dc9525b44_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115409127f696c445a29fb8ca1dc9525b44_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:34,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/90ae244cbd8f4967bf485aa9258a9ae0, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:34,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/90ae244cbd8f4967bf485aa9258a9ae0 is 175, key is test_row_0/A:col10/1731631593107/Put/seqid=0 2024-11-15T00:46:34,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631654159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:34,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:34,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742267_1443 (size=31255) 2024-11-15T00:46:34,169 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=449, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/90ae244cbd8f4967bf485aa9258a9ae0 2024-11-15T00:46:34,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/2ef8699b359a458091009d098eac7b57 is 50, key is test_row_0/B:col10/1731631593107/Put/seqid=0 2024-11-15T00:46:34,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742268_1444 (size=12301) 2024-11-15T00:46:34,206 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/2ef8699b359a458091009d098eac7b57 2024-11-15T00:46:34,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/927c0ce3fb7e4777a9384ab9674774d8 is 50, key is test_row_0/C:col10/1731631593107/Put/seqid=0 2024-11-15T00:46:34,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742269_1445 (size=12301) 2024-11-15T00:46:34,276 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/927c0ce3fb7e4777a9384ab9674774d8 2024-11-15T00:46:34,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/90ae244cbd8f4967bf485aa9258a9ae0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/90ae244cbd8f4967bf485aa9258a9ae0 2024-11-15T00:46:34,296 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/90ae244cbd8f4967bf485aa9258a9ae0, entries=150, sequenceid=449, filesize=30.5 K 2024-11-15T00:46:34,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/2ef8699b359a458091009d098eac7b57 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2ef8699b359a458091009d098eac7b57 2024-11-15T00:46:34,309 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2ef8699b359a458091009d098eac7b57, entries=150, sequenceid=449, filesize=12.0 K 2024-11-15T00:46:34,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/927c0ce3fb7e4777a9384ab9674774d8 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/927c0ce3fb7e4777a9384ab9674774d8 2024-11-15T00:46:34,319 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/927c0ce3fb7e4777a9384ab9674774d8, entries=150, sequenceid=449, filesize=12.0 K 2024-11-15T00:46:34,320 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 587ms, sequenceid=449, compaction requested=false 2024-11-15T00:46:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-15T00:46:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-15T00:46:34,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-15T00:46:34,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9320 sec 2024-11-15T00:46:34,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.9520 sec 2024-11-15T00:46:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:34,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:46:34,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:34,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:34,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:34,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:34,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:34,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:34,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e3a9958e0616401fb7c345e0401862cf_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631594041/Put/seqid=0 2024-11-15T00:46:34,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742270_1446 (size=14994) 2024-11-15T00:46:34,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:34,489 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e3a9958e0616401fb7c345e0401862cf_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e3a9958e0616401fb7c345e0401862cf_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:34,490 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/a6350e11cc5f4e3897b6e9c8154f0703, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:34,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/a6350e11cc5f4e3897b6e9c8154f0703 is 175, key is test_row_0/A:col10/1731631594041/Put/seqid=0 2024-11-15T00:46:34,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631654510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:34,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:34,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60166 deadline: 1731631654519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:34,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60134 deadline: 1731631654519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:34,531 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in ccd4c1fbc17fcc85146b8fbdf8bdc2ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:34,532 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in ccd4c1fbc17fcc85146b8fbdf8bdc2ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:34,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742271_1447 (size=39949) 2024-11-15T00:46:34,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=470, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/a6350e11cc5f4e3897b6e9c8154f0703 2024-11-15T00:46:34,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/f9f3c03b9c5d4ddb8b25f93c701d6f40 is 50, key is test_row_0/B:col10/1731631594041/Put/seqid=0 2024-11-15T00:46:34,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742272_1448 (size=12301) 2024-11-15T00:46:34,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/f9f3c03b9c5d4ddb8b25f93c701d6f40 2024-11-15T00:46:34,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c0d79a960fff46cfbe75f42655dbe2cf is 50, key is test_row_0/C:col10/1731631594041/Put/seqid=0 2024-11-15T00:46:34,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631654629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:34,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742273_1449 (size=12301) 2024-11-15T00:46:34,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c0d79a960fff46cfbe75f42655dbe2cf 2024-11-15T00:46:34,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/a6350e11cc5f4e3897b6e9c8154f0703 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a6350e11cc5f4e3897b6e9c8154f0703 2024-11-15T00:46:34,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a6350e11cc5f4e3897b6e9c8154f0703, entries=200, sequenceid=470, filesize=39.0 K 2024-11-15T00:46:34,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/f9f3c03b9c5d4ddb8b25f93c701d6f40 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f9f3c03b9c5d4ddb8b25f93c701d6f40 2024-11-15T00:46:34,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f9f3c03b9c5d4ddb8b25f93c701d6f40, entries=150, sequenceid=470, filesize=12.0 K 2024-11-15T00:46:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/c0d79a960fff46cfbe75f42655dbe2cf as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c0d79a960fff46cfbe75f42655dbe2cf 2024-11-15T00:46:34,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c0d79a960fff46cfbe75f42655dbe2cf, entries=150, sequenceid=470, filesize=12.0 K 2024-11-15T00:46:34,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 262ms, sequenceid=470, compaction requested=true 2024-11-15T00:46:34,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:34,668 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:34,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:34,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:34,668 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:34,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:34,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:34,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:34,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:34,669 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:34,669 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A is initiating minor compaction (all files) 2024-11-15T00:46:34,669 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:34,669 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b8ae7141e7bf4319803d01eb8b80d24d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/90ae244cbd8f4967bf485aa9258a9ae0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a6350e11cc5f4e3897b6e9c8154f0703] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=101.0 K 2024-11-15T00:46:34,669 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:34,670 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b8ae7141e7bf4319803d01eb8b80d24d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/90ae244cbd8f4967bf485aa9258a9ae0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a6350e11cc5f4e3897b6e9c8154f0703] 2024-11-15T00:46:34,670 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:34,670 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B is initiating minor compaction (all files) 2024-11-15T00:46:34,670 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:34,670 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8ae7141e7bf4319803d01eb8b80d24d, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731631592305 2024-11-15T00:46:34,670 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6f8068f8d0f4466fb48fa0122dcec75c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2ef8699b359a458091009d098eac7b57, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f9f3c03b9c5d4ddb8b25f93c701d6f40] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=37.0 K 2024-11-15T00:46:34,670 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90ae244cbd8f4967bf485aa9258a9ae0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731631593107 2024-11-15T00:46:34,670 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f8068f8d0f4466fb48fa0122dcec75c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731631592305 2024-11-15T00:46:34,670 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6350e11cc5f4e3897b6e9c8154f0703, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731631593973 2024-11-15T00:46:34,671 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef8699b359a458091009d098eac7b57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731631593107 2024-11-15T00:46:34,671 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f9f3c03b9c5d4ddb8b25f93c701d6f40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731631594032 2024-11-15T00:46:34,680 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#B#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:34,680 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/df77fb597fe548c0b37ffbe5bb82d710 is 50, key is test_row_0/B:col10/1731631594041/Put/seqid=0 2024-11-15T00:46:34,683 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:34,691 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115c063dd9e3f114a08b3fc89ea2cd5ffa9_ccd4c1fbc17fcc85146b8fbdf8bdc2ff store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:34,693 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115c063dd9e3f114a08b3fc89ea2cd5ffa9_ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:34,693 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c063dd9e3f114a08b3fc89ea2cd5ffa9_ccd4c1fbc17fcc85146b8fbdf8bdc2ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:34,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742274_1450 (size=13391) 2024-11-15T00:46:34,708 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/df77fb597fe548c0b37ffbe5bb82d710 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/df77fb597fe548c0b37ffbe5bb82d710 2024-11-15T00:46:34,714 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into df77fb597fe548c0b37ffbe5bb82d710(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:34,715 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:34,715 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, priority=13, startTime=1731631594668; duration=0sec 2024-11-15T00:46:34,715 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:34,715 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:34,715 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:34,716 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:34,716 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C is initiating minor compaction (all files) 2024-11-15T00:46:34,716 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C in TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:34,716 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/28b7660d5e5c4ce8a4158c62e3ed467d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/927c0ce3fb7e4777a9384ab9674774d8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c0d79a960fff46cfbe75f42655dbe2cf] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp, totalSize=37.0 K 2024-11-15T00:46:34,716 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 28b7660d5e5c4ce8a4158c62e3ed467d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731631592305 2024-11-15T00:46:34,720 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 927c0ce3fb7e4777a9384ab9674774d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731631593107 2024-11-15T00:46:34,721 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d79a960fff46cfbe75f42655dbe2cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731631594032 2024-11-15T00:46:34,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742275_1451 (size=4469) 2024-11-15T00:46:34,729 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#A#compaction#372 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:34,730 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c8d556ffd51348fab6d9e6c811995681 is 175, key is test_row_0/A:col10/1731631594041/Put/seqid=0 2024-11-15T00:46:34,734 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4c1fbc17fcc85146b8fbdf8bdc2ff#C#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:34,735 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/f64dc6688b794e709adba3c2b6c2b8e2 is 50, key is test_row_0/C:col10/1731631594041/Put/seqid=0 2024-11-15T00:46:34,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742277_1453 (size=13391) 2024-11-15T00:46:34,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742276_1452 (size=32345) 2024-11-15T00:46:34,771 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/f64dc6688b794e709adba3c2b6c2b8e2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/f64dc6688b794e709adba3c2b6c2b8e2 2024-11-15T00:46:34,780 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/c8d556ffd51348fab6d9e6c811995681 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c8d556ffd51348fab6d9e6c811995681 2024-11-15T00:46:34,790 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into c8d556ffd51348fab6d9e6c811995681(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:34,790 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:34,790 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, priority=13, startTime=1731631594668; duration=0sec 2024-11-15T00:46:34,791 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C of ccd4c1fbc17fcc85146b8fbdf8bdc2ff into f64dc6688b794e709adba3c2b6c2b8e2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:34,791 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:34,791 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., storeName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, priority=13, startTime=1731631594669; duration=0sec 2024-11-15T00:46:34,791 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:34,791 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:34,792 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:34,792 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:34,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:46:34,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:34,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:34,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:34,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:34,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:34,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:34,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157c044032793b4ee2904f07d64323e3f8_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631594508/Put/seqid=0 2024-11-15T00:46:34,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742278_1454 (size=14994) 2024-11-15T00:46:35,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631655023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631655026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:35,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:35,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631655149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:35,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:35,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631655149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:35,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:35,350 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:35,361 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157c044032793b4ee2904f07d64323e3f8_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c044032793b4ee2904f07d64323e3f8_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:35,362 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/771c2053b5034b8b92453fc9cc4de2e9, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:35,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/771c2053b5034b8b92453fc9cc4de2e9 is 175, key is test_row_0/A:col10/1731631594508/Put/seqid=0 2024-11-15T00:46:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631655359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631655359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:35,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742279_1455 (size=39949) 2024-11-15T00:46:35,385 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=486, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/771c2053b5034b8b92453fc9cc4de2e9 2024-11-15T00:46:35,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/7627a346407342879a6ecd87c5cd1d8f is 50, key is test_row_0/B:col10/1731631594508/Put/seqid=0 2024-11-15T00:46:35,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742280_1456 (size=12301) 2024-11-15T00:46:35,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/7627a346407342879a6ecd87c5cd1d8f 2024-11-15T00:46:35,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/5e12ed957db94e0c88039ef03fcfda14 is 50, key is test_row_0/C:col10/1731631594508/Put/seqid=0 2024-11-15T00:46:35,459 INFO [Thread-1498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1498. 2024-11-15T00:46:35,459 DEBUG [Thread-1498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:35,460 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:35,460 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:35,461 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:35,465 INFO [Thread-1490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1490. 2024-11-15T00:46:35,465 DEBUG [Thread-1490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:35,465 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:35,465 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:35,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:35,468 INFO [Thread-1492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1492. 2024-11-15T00:46:35,468 DEBUG [Thread-1492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:35,468 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:35,468 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:35,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:35,473 INFO [Thread-1494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1494. 2024-11-15T00:46:35,473 DEBUG [Thread-1494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:35,473 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:35,473 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:35,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:35,479 INFO [Thread-1496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1496. 2024-11-15T00:46:35,479 DEBUG [Thread-1496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:35,479 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:35,479 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:35,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:35,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742281_1457 (size=12301) 2024-11-15T00:46:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:46:35,560 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:35,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60180 deadline: 1731631655670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:35,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:35,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60188 deadline: 1731631655679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:35,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:35,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:46:35,699 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2049c710453e4d1ea49af3492af1619f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9dc40ff70e434498b0807bbb0d09d28b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/354ce60027c14c51901357412155f22e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/72359b737ffc44b099ea9de0c302f820, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/173f9ca917ab450a8b4d01e9fa0cf63c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2a8d4345301445cd86d6cf7dbd01245d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f3faf79e7cd044d2bfa46c15d5c7acda, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/be6eb3e3a37842109fef8d7b670bf05a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/626eba4d66894ec3b1277dd29ca5b71b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/acb789889e8c468ca8cdbbd571fc9faf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a097cdf9ee70446fb460d5f6645ebf0c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/6fda998aac78446e94bc68d4d160ea37, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b00742421f0c4aee9cece17f060a6997, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c04b7ec8601844e1b66fab3e3ff25613, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/ce172e07a42e4b9c97710a02954453db, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d1cbae21d744ad28947b94c7cc08a00, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e8635d6b77a148fbb81b01bbd7e3cd2f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e0400c2c27c74a0b841b9797fcd8e5b3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c5775bbb9b504638acc11e015a0e45a4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d75ad3601dd401a863aefe212cbdd59, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/d19102a48adb407995c82cf69843a93e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/28cdf7cac4e646d9b5824af99f9e81b7, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9d71a2f39a1b40edaf7dc366c3161286, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/fd78430efa67459897fc5ca1338b88ce, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c6b337c67af141aa9d88f039a63fbf7f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a33270521613499db099ca57dc8b10ac, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f5cdd3c70ea84d1990b09a017371b70c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c4347dba47404509abee595b8a3fc51b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/627147f950394d69906d82f1a6d7bf90, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b8ae7141e7bf4319803d01eb8b80d24d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/90ae244cbd8f4967bf485aa9258a9ae0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a6350e11cc5f4e3897b6e9c8154f0703] to archive 2024-11-15T00:46:35,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:46:35,702 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2049c710453e4d1ea49af3492af1619f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2049c710453e4d1ea49af3492af1619f 2024-11-15T00:46:35,703 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9dc40ff70e434498b0807bbb0d09d28b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9dc40ff70e434498b0807bbb0d09d28b 2024-11-15T00:46:35,704 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/354ce60027c14c51901357412155f22e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/354ce60027c14c51901357412155f22e 2024-11-15T00:46:35,705 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/72359b737ffc44b099ea9de0c302f820 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/72359b737ffc44b099ea9de0c302f820 2024-11-15T00:46:35,706 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/173f9ca917ab450a8b4d01e9fa0cf63c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/173f9ca917ab450a8b4d01e9fa0cf63c 2024-11-15T00:46:35,708 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2a8d4345301445cd86d6cf7dbd01245d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/2a8d4345301445cd86d6cf7dbd01245d 2024-11-15T00:46:35,709 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f3faf79e7cd044d2bfa46c15d5c7acda to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f3faf79e7cd044d2bfa46c15d5c7acda 2024-11-15T00:46:35,710 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/be6eb3e3a37842109fef8d7b670bf05a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/be6eb3e3a37842109fef8d7b670bf05a 2024-11-15T00:46:35,712 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/626eba4d66894ec3b1277dd29ca5b71b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/626eba4d66894ec3b1277dd29ca5b71b 2024-11-15T00:46:35,715 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/acb789889e8c468ca8cdbbd571fc9faf to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/acb789889e8c468ca8cdbbd571fc9faf 2024-11-15T00:46:35,716 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a097cdf9ee70446fb460d5f6645ebf0c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a097cdf9ee70446fb460d5f6645ebf0c 2024-11-15T00:46:35,721 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/6fda998aac78446e94bc68d4d160ea37 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/6fda998aac78446e94bc68d4d160ea37 2024-11-15T00:46:35,722 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b00742421f0c4aee9cece17f060a6997 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b00742421f0c4aee9cece17f060a6997 2024-11-15T00:46:35,723 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c04b7ec8601844e1b66fab3e3ff25613 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c04b7ec8601844e1b66fab3e3ff25613 2024-11-15T00:46:35,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/ce172e07a42e4b9c97710a02954453db to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/ce172e07a42e4b9c97710a02954453db 2024-11-15T00:46:35,726 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d1cbae21d744ad28947b94c7cc08a00 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d1cbae21d744ad28947b94c7cc08a00 2024-11-15T00:46:35,727 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e8635d6b77a148fbb81b01bbd7e3cd2f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e8635d6b77a148fbb81b01bbd7e3cd2f 2024-11-15T00:46:35,728 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e0400c2c27c74a0b841b9797fcd8e5b3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/e0400c2c27c74a0b841b9797fcd8e5b3 2024-11-15T00:46:35,729 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c5775bbb9b504638acc11e015a0e45a4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c5775bbb9b504638acc11e015a0e45a4 2024-11-15T00:46:35,730 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d75ad3601dd401a863aefe212cbdd59 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8d75ad3601dd401a863aefe212cbdd59 2024-11-15T00:46:35,731 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/d19102a48adb407995c82cf69843a93e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/d19102a48adb407995c82cf69843a93e 2024-11-15T00:46:35,733 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/28cdf7cac4e646d9b5824af99f9e81b7 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/28cdf7cac4e646d9b5824af99f9e81b7 2024-11-15T00:46:35,734 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9d71a2f39a1b40edaf7dc366c3161286 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/9d71a2f39a1b40edaf7dc366c3161286 2024-11-15T00:46:35,735 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/fd78430efa67459897fc5ca1338b88ce to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/fd78430efa67459897fc5ca1338b88ce 2024-11-15T00:46:35,735 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c6b337c67af141aa9d88f039a63fbf7f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c6b337c67af141aa9d88f039a63fbf7f 2024-11-15T00:46:35,737 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a33270521613499db099ca57dc8b10ac to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a33270521613499db099ca57dc8b10ac 2024-11-15T00:46:35,738 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f5cdd3c70ea84d1990b09a017371b70c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/f5cdd3c70ea84d1990b09a017371b70c 2024-11-15T00:46:35,739 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c4347dba47404509abee595b8a3fc51b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c4347dba47404509abee595b8a3fc51b 2024-11-15T00:46:35,740 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/627147f950394d69906d82f1a6d7bf90 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/627147f950394d69906d82f1a6d7bf90 2024-11-15T00:46:35,741 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b8ae7141e7bf4319803d01eb8b80d24d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/b8ae7141e7bf4319803d01eb8b80d24d 2024-11-15T00:46:35,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/90ae244cbd8f4967bf485aa9258a9ae0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/90ae244cbd8f4967bf485aa9258a9ae0 2024-11-15T00:46:35,744 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a6350e11cc5f4e3897b6e9c8154f0703 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/a6350e11cc5f4e3897b6e9c8154f0703 2024-11-15T00:46:35,748 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bdf76800a989406786f7400ce6a4b762, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/c64c7abb3f8d4402b03a77d38fdef28c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2c1c5663cb0d413ab103c8946ce3f593, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bf46fa6f96ca42149460fae9d5a7856b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/971ad0a7e27544b2a396c78710780c4e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/96e51a4824fa4d0583bc408b7d3ab741, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3576d8b7d36247f0a7a8c7b180a136f6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/63061d01510b4dcfab0e1d27a9d54b34, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f6907340a33746f1bdbe2734e38fe3a3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f330d53c36174b7f83e75adbecd2ed7f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3c5e38c823c2401da30f22bf41a7a160, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/7d582d9252cb4a1d8bad2184f6c1740c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/66baac4a4c914adb922dea6852138c82, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/e1a84f0bd6f644ac9430996f95e92c81, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/dcf9fa3c204b43aabeb173e21ecb1005, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6cdfe4c500d646829db42a3c20a75bbb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a2c8f7c6e9964990b81ace5a45f22c96, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/b41aa498e7f44beb90b075add5736caf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/20d20505ae1e4f2a85f6f896746cd3fd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/1745d816c43b42f0aa13586710429b12, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a85f79be6d0648b7ab2ed8e21af32569, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/9922684cf8454dc092487f2c28a03ec0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/88a22a8dfaa84adcaf1f571f8814d97a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ec768df608b543e5a0c6a5323efe60eb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ac9dd99689854f9381d11805fd9759bb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/dc953eafb2de4cc394948b11768148cd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/4171f314cc864fea9c6d63bb204b1e90, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/41b48ea4a6b843878667e0ce07a0ea27, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6f8068f8d0f4466fb48fa0122dcec75c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/0dc3846cd4cf49e09c2c148a3c1f751b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2ef8699b359a458091009d098eac7b57, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f9f3c03b9c5d4ddb8b25f93c701d6f40] to archive 2024-11-15T00:46:35,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:46:35,750 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bdf76800a989406786f7400ce6a4b762 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bdf76800a989406786f7400ce6a4b762 2024-11-15T00:46:35,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/c64c7abb3f8d4402b03a77d38fdef28c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/c64c7abb3f8d4402b03a77d38fdef28c 2024-11-15T00:46:35,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2c1c5663cb0d413ab103c8946ce3f593 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2c1c5663cb0d413ab103c8946ce3f593 2024-11-15T00:46:35,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bf46fa6f96ca42149460fae9d5a7856b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/bf46fa6f96ca42149460fae9d5a7856b 2024-11-15T00:46:35,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/971ad0a7e27544b2a396c78710780c4e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/971ad0a7e27544b2a396c78710780c4e 2024-11-15T00:46:35,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/96e51a4824fa4d0583bc408b7d3ab741 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/96e51a4824fa4d0583bc408b7d3ab741 2024-11-15T00:46:35,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3576d8b7d36247f0a7a8c7b180a136f6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3576d8b7d36247f0a7a8c7b180a136f6 2024-11-15T00:46:35,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/63061d01510b4dcfab0e1d27a9d54b34 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/63061d01510b4dcfab0e1d27a9d54b34 2024-11-15T00:46:35,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f6907340a33746f1bdbe2734e38fe3a3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f6907340a33746f1bdbe2734e38fe3a3 2024-11-15T00:46:35,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f330d53c36174b7f83e75adbecd2ed7f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f330d53c36174b7f83e75adbecd2ed7f 2024-11-15T00:46:35,761 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3c5e38c823c2401da30f22bf41a7a160 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/3c5e38c823c2401da30f22bf41a7a160 2024-11-15T00:46:35,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/7d582d9252cb4a1d8bad2184f6c1740c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/7d582d9252cb4a1d8bad2184f6c1740c 2024-11-15T00:46:35,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/66baac4a4c914adb922dea6852138c82 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/66baac4a4c914adb922dea6852138c82 2024-11-15T00:46:35,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/e1a84f0bd6f644ac9430996f95e92c81 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/e1a84f0bd6f644ac9430996f95e92c81 2024-11-15T00:46:35,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/dcf9fa3c204b43aabeb173e21ecb1005 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/dcf9fa3c204b43aabeb173e21ecb1005 2024-11-15T00:46:35,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6cdfe4c500d646829db42a3c20a75bbb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6cdfe4c500d646829db42a3c20a75bbb 2024-11-15T00:46:35,767 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a2c8f7c6e9964990b81ace5a45f22c96 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a2c8f7c6e9964990b81ace5a45f22c96 2024-11-15T00:46:35,768 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/b41aa498e7f44beb90b075add5736caf to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/b41aa498e7f44beb90b075add5736caf 2024-11-15T00:46:35,769 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/20d20505ae1e4f2a85f6f896746cd3fd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/20d20505ae1e4f2a85f6f896746cd3fd 2024-11-15T00:46:35,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/1745d816c43b42f0aa13586710429b12 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/1745d816c43b42f0aa13586710429b12 2024-11-15T00:46:35,771 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a85f79be6d0648b7ab2ed8e21af32569 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a85f79be6d0648b7ab2ed8e21af32569 2024-11-15T00:46:35,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/9922684cf8454dc092487f2c28a03ec0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/9922684cf8454dc092487f2c28a03ec0 2024-11-15T00:46:35,775 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/88a22a8dfaa84adcaf1f571f8814d97a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/88a22a8dfaa84adcaf1f571f8814d97a 2024-11-15T00:46:35,776 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ec768df608b543e5a0c6a5323efe60eb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ec768df608b543e5a0c6a5323efe60eb 2024-11-15T00:46:35,777 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ac9dd99689854f9381d11805fd9759bb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/ac9dd99689854f9381d11805fd9759bb 2024-11-15T00:46:35,778 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/dc953eafb2de4cc394948b11768148cd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/dc953eafb2de4cc394948b11768148cd 2024-11-15T00:46:35,779 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/4171f314cc864fea9c6d63bb204b1e90 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/4171f314cc864fea9c6d63bb204b1e90 2024-11-15T00:46:35,780 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/41b48ea4a6b843878667e0ce07a0ea27 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/41b48ea4a6b843878667e0ce07a0ea27 2024-11-15T00:46:35,782 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6f8068f8d0f4466fb48fa0122dcec75c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/6f8068f8d0f4466fb48fa0122dcec75c 2024-11-15T00:46:35,783 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/0dc3846cd4cf49e09c2c148a3c1f751b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/0dc3846cd4cf49e09c2c148a3c1f751b 2024-11-15T00:46:35,784 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2ef8699b359a458091009d098eac7b57 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/2ef8699b359a458091009d098eac7b57 2024-11-15T00:46:35,791 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f9f3c03b9c5d4ddb8b25f93c701d6f40 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/f9f3c03b9c5d4ddb8b25f93c701d6f40 2024-11-15T00:46:35,795 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/cdc733889dc947bbb86f72f0b016d211, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/08a51efbe11e46c78039d52246abb3ee, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/4e18fb049bb64cb9aa3694db9306ce20, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/aceb50aa1e9a409ea903eee29abf888b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/924d7792fd864d3790f4ef9a787c9cb3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/242a3525ea4046cd8841614dfae8b065, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/53e9e007b3f54b3d84db014fec2ebcec, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/272a61d92081406da4e0f59ffd4e1b84, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c567918083754c43a645a0572ce2e712, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c894759d2c744551a9748a14323b40a0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/102f85389af04fd8bded5cfeaf25265c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/b40953f5d3e54528adb2d05c5047d3b9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e99827d472094f169f953d33c41af478, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d9648597b52b45d098bed45f477b23d3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c5c137f0fe2f453c94d9cd74fc8212dd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/04fe84e67e9b48e48be62035dd976fbd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/49c524383d304a03ab1db0a90025c08f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ef8174b20d324602b42f47e8ddd8bdf8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/dff1c17032ce4508a2eaa7617f81e500, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e2693e0e26e742ab95ff302f4c6e1559, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d03d5fff32574eca90d0d6bcd4327c48, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/0c4a417df2f64fd6be25b5703020568d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/95d1883d73654fe9aaa60f43a3321434, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ddde1e21810947179c88970e0cbdf926, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c044da4ce3ad4399812609a0ab68d6e2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/fccd52e1a72f42d4b93a07644bb635c3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/72346911dc9a4b25b8f07e669f033963, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ea9606f470d24e8db59b636fce3943d4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/28b7660d5e5c4ce8a4158c62e3ed467d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/643206632cd0452296af6fd5326f6a73, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/927c0ce3fb7e4777a9384ab9674774d8, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c0d79a960fff46cfbe75f42655dbe2cf] to archive 2024-11-15T00:46:35,796 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:46:35,797 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/cdc733889dc947bbb86f72f0b016d211 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/cdc733889dc947bbb86f72f0b016d211 2024-11-15T00:46:35,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/08a51efbe11e46c78039d52246abb3ee to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/08a51efbe11e46c78039d52246abb3ee 2024-11-15T00:46:35,799 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/4e18fb049bb64cb9aa3694db9306ce20 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/4e18fb049bb64cb9aa3694db9306ce20 2024-11-15T00:46:35,800 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/aceb50aa1e9a409ea903eee29abf888b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/aceb50aa1e9a409ea903eee29abf888b 2024-11-15T00:46:35,801 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/924d7792fd864d3790f4ef9a787c9cb3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/924d7792fd864d3790f4ef9a787c9cb3 2024-11-15T00:46:35,802 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/242a3525ea4046cd8841614dfae8b065 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/242a3525ea4046cd8841614dfae8b065 2024-11-15T00:46:35,803 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/53e9e007b3f54b3d84db014fec2ebcec to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/53e9e007b3f54b3d84db014fec2ebcec 2024-11-15T00:46:35,803 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/272a61d92081406da4e0f59ffd4e1b84 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/272a61d92081406da4e0f59ffd4e1b84 2024-11-15T00:46:35,804 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c567918083754c43a645a0572ce2e712 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c567918083754c43a645a0572ce2e712 2024-11-15T00:46:35,805 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c894759d2c744551a9748a14323b40a0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c894759d2c744551a9748a14323b40a0 2024-11-15T00:46:35,806 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/102f85389af04fd8bded5cfeaf25265c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/102f85389af04fd8bded5cfeaf25265c 2024-11-15T00:46:35,807 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/b40953f5d3e54528adb2d05c5047d3b9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/b40953f5d3e54528adb2d05c5047d3b9 2024-11-15T00:46:35,808 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e99827d472094f169f953d33c41af478 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e99827d472094f169f953d33c41af478 2024-11-15T00:46:35,809 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d9648597b52b45d098bed45f477b23d3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d9648597b52b45d098bed45f477b23d3 2024-11-15T00:46:35,810 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c5c137f0fe2f453c94d9cd74fc8212dd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c5c137f0fe2f453c94d9cd74fc8212dd 2024-11-15T00:46:35,811 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/04fe84e67e9b48e48be62035dd976fbd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/04fe84e67e9b48e48be62035dd976fbd 2024-11-15T00:46:35,812 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/49c524383d304a03ab1db0a90025c08f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/49c524383d304a03ab1db0a90025c08f 2024-11-15T00:46:35,813 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ef8174b20d324602b42f47e8ddd8bdf8 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ef8174b20d324602b42f47e8ddd8bdf8 2024-11-15T00:46:35,814 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/dff1c17032ce4508a2eaa7617f81e500 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/dff1c17032ce4508a2eaa7617f81e500 2024-11-15T00:46:35,815 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e2693e0e26e742ab95ff302f4c6e1559 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/e2693e0e26e742ab95ff302f4c6e1559 2024-11-15T00:46:35,816 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d03d5fff32574eca90d0d6bcd4327c48 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/d03d5fff32574eca90d0d6bcd4327c48 2024-11-15T00:46:35,817 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/0c4a417df2f64fd6be25b5703020568d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/0c4a417df2f64fd6be25b5703020568d 2024-11-15T00:46:35,819 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/95d1883d73654fe9aaa60f43a3321434 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/95d1883d73654fe9aaa60f43a3321434 2024-11-15T00:46:35,821 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ddde1e21810947179c88970e0cbdf926 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ddde1e21810947179c88970e0cbdf926 2024-11-15T00:46:35,822 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c044da4ce3ad4399812609a0ab68d6e2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c044da4ce3ad4399812609a0ab68d6e2 2024-11-15T00:46:35,823 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/fccd52e1a72f42d4b93a07644bb635c3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/fccd52e1a72f42d4b93a07644bb635c3 2024-11-15T00:46:35,824 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/72346911dc9a4b25b8f07e669f033963 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/72346911dc9a4b25b8f07e669f033963 2024-11-15T00:46:35,825 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ea9606f470d24e8db59b636fce3943d4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/ea9606f470d24e8db59b636fce3943d4 2024-11-15T00:46:35,827 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/28b7660d5e5c4ce8a4158c62e3ed467d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/28b7660d5e5c4ce8a4158c62e3ed467d 2024-11-15T00:46:35,828 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/643206632cd0452296af6fd5326f6a73 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/643206632cd0452296af6fd5326f6a73 2024-11-15T00:46:35,830 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/927c0ce3fb7e4777a9384ab9674774d8 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/927c0ce3fb7e4777a9384ab9674774d8 2024-11-15T00:46:35,832 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c0d79a960fff46cfbe75f42655dbe2cf to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/c0d79a960fff46cfbe75f42655dbe2cf 2024-11-15T00:46:35,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/5e12ed957db94e0c88039ef03fcfda14 2024-11-15T00:46:35,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/771c2053b5034b8b92453fc9cc4de2e9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/771c2053b5034b8b92453fc9cc4de2e9 2024-11-15T00:46:35,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/771c2053b5034b8b92453fc9cc4de2e9, entries=200, sequenceid=486, filesize=39.0 K 2024-11-15T00:46:35,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/7627a346407342879a6ecd87c5cd1d8f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/7627a346407342879a6ecd87c5cd1d8f 2024-11-15T00:46:35,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/7627a346407342879a6ecd87c5cd1d8f, entries=150, sequenceid=486, filesize=12.0 K 2024-11-15T00:46:35,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/5e12ed957db94e0c88039ef03fcfda14 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/5e12ed957db94e0c88039ef03fcfda14 2024-11-15T00:46:35,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/5e12ed957db94e0c88039ef03fcfda14, entries=150, sequenceid=486, filesize=12.0 K 2024-11-15T00:46:35,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 1028ms, sequenceid=486, compaction requested=false 2024-11-15T00:46:35,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:36,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:46:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:36,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:36,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:36,185 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-11-15T00:46:36,185 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:36,185 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,185 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,186 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:36,196 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-11-15T00:46:36,196 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:36,196 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,196 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,197 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:36,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cf206b1937844709b4e13c1a2b5a55d4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631596180/Put/seqid=0 2024-11-15T00:46:36,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742282_1458 (size=12454) 2024-11-15T00:46:36,550 INFO [Thread-1479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1479. 2024-11-15T00:46:36,550 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-11-15T00:46:36,550 DEBUG [Thread-1479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:36,550 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,551 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:36,551 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,551 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:36,551 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:36,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:36,638 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cf206b1937844709b4e13c1a2b5a55d4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cf206b1937844709b4e13c1a2b5a55d4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:36,639 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/19beb197467c46008cf01e7952913b93, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:36,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/19beb197467c46008cf01e7952913b93 is 175, key is test_row_0/A:col10/1731631596180/Put/seqid=0 2024-11-15T00:46:36,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742283_1459 (size=31255) 2024-11-15T00:46:36,673 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=506, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/19beb197467c46008cf01e7952913b93 2024-11-15T00:46:36,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/a4136a177d984b6586774047cb777cb1 is 50, key is test_row_0/B:col10/1731631596180/Put/seqid=0 2024-11-15T00:46:36,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742284_1460 (size=12301) 2024-11-15T00:46:36,991 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-11-15T00:46:36,991 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:46:36,991 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,991 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 145 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 836 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2499 rows 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 822 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2461 rows 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 830 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2489 rows 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 815 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2436 rows 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 823 2024-11-15T00:46:36,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2469 rows 2024-11-15T00:46:36,992 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:46:36,992 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:46:36,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:36,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:46:37,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:46:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:46:37,007 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631597007"}]},"ts":"1731631597007"} 2024-11-15T00:46:37,009 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:46:37,009 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:46:37,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:46:37,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, UNASSIGN}] 2024-11-15T00:46:37,014 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, UNASSIGN 2024-11-15T00:46:37,016 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, regionState=CLOSING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:37,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, UNASSIGN because future has completed 2024-11-15T00:46:37,018 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:46:37,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:46:37,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/a4136a177d984b6586774047cb777cb1 2024-11-15T00:46:37,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/5e804a9c80d8479eba82c6c798f75e5d is 50, key is test_row_0/C:col10/1731631596180/Put/seqid=0 2024-11-15T00:46:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:46:37,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742285_1461 (size=12301) 2024-11-15T00:46:37,171 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:37,171 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:46:37,171 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing ccd4c1fbc17fcc85146b8fbdf8bdc2ff, disabling compactions & flushes 2024-11-15T00:46:37,172 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:46:37,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/5e804a9c80d8479eba82c6c798f75e5d 2024-11-15T00:46:37,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/19beb197467c46008cf01e7952913b93 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/19beb197467c46008cf01e7952913b93 2024-11-15T00:46:37,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/19beb197467c46008cf01e7952913b93, entries=150, sequenceid=506, filesize=30.5 K 2024-11-15T00:46:37,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/a4136a177d984b6586774047cb777cb1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a4136a177d984b6586774047cb777cb1 2024-11-15T00:46:37,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a4136a177d984b6586774047cb777cb1, entries=150, sequenceid=506, filesize=12.0 K 2024-11-15T00:46:37,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/5e804a9c80d8479eba82c6c798f75e5d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/5e804a9c80d8479eba82c6c798f75e5d 2024-11-15T00:46:37,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/5e804a9c80d8479eba82c6c798f75e5d, entries=150, sequenceid=506, filesize=12.0 K 2024-11-15T00:46:37,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=26.84 KB/27480 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 1345ms, sequenceid=506, compaction requested=true 2024-11-15T00:46:37,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: 2024-11-15T00:46:37,529 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:37,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:37,529 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:37,529 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. because compaction request was cancelled 2024-11-15T00:46:37,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:37,529 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. after waiting 0 ms 2024-11-15T00:46:37,529 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:A 2024-11-15T00:46:37,529 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:37,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:37,529 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. because compaction request was cancelled 2024-11-15T00:46:37,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:37,529 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:B 2024-11-15T00:46:37,529 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. because compaction request was cancelled 2024-11-15T00:46:37,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:37,529 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4c1fbc17fcc85146b8fbdf8bdc2ff:C 2024-11-15T00:46:37,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:37,529 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-15T00:46:37,529 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=A 2024-11-15T00:46:37,529 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:37,529 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=B 2024-11-15T00:46:37,530 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:37,530 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4c1fbc17fcc85146b8fbdf8bdc2ff, store=C 2024-11-15T00:46:37,530 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:37,534 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115750463dd1dc14ee89f81c71b36e93ddc_ccd4c1fbc17fcc85146b8fbdf8bdc2ff is 50, key is test_row_0/A:col10/1731631596549/Put/seqid=0 2024-11-15T00:46:37,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742286_1462 (size=9914) 2024-11-15T00:46:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:46:37,937 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:37,940 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115750463dd1dc14ee89f81c71b36e93ddc_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115750463dd1dc14ee89f81c71b36e93ddc_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:37,941 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8296d3ef9b4f444fb4391433167f0671, store: [table=TestAcidGuarantees family=A region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff] 2024-11-15T00:46:37,941 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8296d3ef9b4f444fb4391433167f0671 is 175, key is test_row_0/A:col10/1731631596549/Put/seqid=0 2024-11-15T00:46:37,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742287_1463 (size=22561) 2024-11-15T00:46:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:46:38,344 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=513, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8296d3ef9b4f444fb4391433167f0671 2024-11-15T00:46:38,353 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/06b684eeea584a77a0b106b8ceaf5596 is 50, key is test_row_0/B:col10/1731631596549/Put/seqid=0 2024-11-15T00:46:38,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742288_1464 (size=9857) 2024-11-15T00:46:38,761 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/06b684eeea584a77a0b106b8ceaf5596 2024-11-15T00:46:38,826 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/fbe1746bad8941ecbc10f0587375059e is 50, key is test_row_0/C:col10/1731631596549/Put/seqid=0 2024-11-15T00:46:38,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742289_1465 (size=9857) 2024-11-15T00:46:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:46:39,255 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/fbe1746bad8941ecbc10f0587375059e 2024-11-15T00:46:39,262 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/A/8296d3ef9b4f444fb4391433167f0671 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8296d3ef9b4f444fb4391433167f0671 2024-11-15T00:46:39,274 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8296d3ef9b4f444fb4391433167f0671, entries=100, sequenceid=513, filesize=22.0 K 2024-11-15T00:46:39,278 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/B/06b684eeea584a77a0b106b8ceaf5596 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/06b684eeea584a77a0b106b8ceaf5596 2024-11-15T00:46:39,302 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/06b684eeea584a77a0b106b8ceaf5596, entries=100, sequenceid=513, filesize=9.6 K 2024-11-15T00:46:39,308 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/.tmp/C/fbe1746bad8941ecbc10f0587375059e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/fbe1746bad8941ecbc10f0587375059e 2024-11-15T00:46:39,346 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/fbe1746bad8941ecbc10f0587375059e, entries=100, sequenceid=513, filesize=9.6 K 2024-11-15T00:46:39,347 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 1818ms, sequenceid=513, compaction requested=true 2024-11-15T00:46:39,378 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/recovered.edits/516.seqid, newMaxSeqId=516, maxSeqId=4 2024-11-15T00:46:39,383 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. 2024-11-15T00:46:39,384 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for ccd4c1fbc17fcc85146b8fbdf8bdc2ff: Waiting for close lock at 1731631597171Running coprocessor pre-close hooks at 1731631597171Disabling compacts and flushes for region at 1731631597171Disabling writes for close at 1731631597529 (+358 ms)Obtaining lock to block concurrent updates at 1731631597529Preparing flush snapshotting stores in ccd4c1fbc17fcc85146b8fbdf8bdc2ff at 1731631597529Finished memstore snapshotting TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731631597530 (+1 ms)Flushing stores of TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff. at 1731631597530Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A: creating writer at 1731631597530Flushing mob file ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A: appending metadata at 1731631597534 (+4 ms)Flushing mob file ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A: closing flushed file at 1731631597534Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A: appending metadata at 1731631597941 (+407 ms)Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A: closing flushed file at 1731631597941Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B: creating writer at 1731631598347 (+406 ms)Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B: appending metadata at 1731631598353 (+6 ms)Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B: closing flushed file at 1731631598353Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C: creating writer at 1731631598806 (+453 ms)Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C: appending metadata at 1731631598825 (+19 ms)Flushing ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C: closing flushed file at 1731631598825Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31971c0c: reopening flushed file at 1731631599260 (+435 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30dea33b: reopening flushed file at 1731631599274 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71726363: reopening flushed file at 1731631599302 (+28 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ccd4c1fbc17fcc85146b8fbdf8bdc2ff in 1818ms, sequenceid=513, compaction requested=true at 1731631599347 (+45 ms)Writing region close event to WAL at 1731631599357 (+10 ms)Running coprocessor post-close hooks at 1731631599383 (+26 ms)Closed at 1731631599383 2024-11-15T00:46:39,387 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, regionState=CLOSED 2024-11-15T00:46:39,388 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:39,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:46:39,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-15T00:46:39,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure ccd4c1fbc17fcc85146b8fbdf8bdc2ff, server=fb2f6cb7846a,34537,1731631472208 in 2.3750 sec 2024-11-15T00:46:39,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-15T00:46:39,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4c1fbc17fcc85146b8fbdf8bdc2ff, UNASSIGN in 2.3870 sec 2024-11-15T00:46:39,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-15T00:46:39,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3940 sec 2024-11-15T00:46:39,409 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631599409"}]},"ts":"1731631599409"} 2024-11-15T00:46:39,411 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:46:39,411 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:46:39,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4160 sec 2024-11-15T00:46:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:46:41,160 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:41,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:46:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:41,162 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T00:46:41,163 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:41,166 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,168 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/recovered.edits] 2024-11-15T00:46:41,176 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/19beb197467c46008cf01e7952913b93 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/19beb197467c46008cf01e7952913b93 2024-11-15T00:46:41,178 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/771c2053b5034b8b92453fc9cc4de2e9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/771c2053b5034b8b92453fc9cc4de2e9 2024-11-15T00:46:41,179 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8296d3ef9b4f444fb4391433167f0671 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/8296d3ef9b4f444fb4391433167f0671 2024-11-15T00:46:41,181 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c8d556ffd51348fab6d9e6c811995681 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/A/c8d556ffd51348fab6d9e6c811995681 2024-11-15T00:46:41,198 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/06b684eeea584a77a0b106b8ceaf5596 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/06b684eeea584a77a0b106b8ceaf5596 2024-11-15T00:46:41,199 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/7627a346407342879a6ecd87c5cd1d8f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/7627a346407342879a6ecd87c5cd1d8f 2024-11-15T00:46:41,200 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a4136a177d984b6586774047cb777cb1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/a4136a177d984b6586774047cb777cb1 2024-11-15T00:46:41,201 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/df77fb597fe548c0b37ffbe5bb82d710 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/B/df77fb597fe548c0b37ffbe5bb82d710 2024-11-15T00:46:41,203 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/5e12ed957db94e0c88039ef03fcfda14 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/5e12ed957db94e0c88039ef03fcfda14 2024-11-15T00:46:41,204 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/5e804a9c80d8479eba82c6c798f75e5d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/5e804a9c80d8479eba82c6c798f75e5d 2024-11-15T00:46:41,206 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/f64dc6688b794e709adba3c2b6c2b8e2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/f64dc6688b794e709adba3c2b6c2b8e2 2024-11-15T00:46:41,207 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/fbe1746bad8941ecbc10f0587375059e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/C/fbe1746bad8941ecbc10f0587375059e 2024-11-15T00:46:41,209 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/recovered.edits/516.seqid to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff/recovered.edits/516.seqid 2024-11-15T00:46:41,210 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,210 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:46:41,211 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:46:41,212 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T00:46:41,215 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150875da7e1c424344a00c90429b2fe7d5_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150875da7e1c424344a00c90429b2fe7d5_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,216 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d2d17e54ccc4ba893252e678cf111b2_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d2d17e54ccc4ba893252e678cf111b2_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,217 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150f5d39b0321943af88136d2734efde99_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150f5d39b0321943af88136d2734efde99_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,218 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115125e50e71ec04d45bd3019334b9bbb43_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115125e50e71ec04d45bd3019334b9bbb43_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,219 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111513177b499d44483baabd33797ebc62b0_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111513177b499d44483baabd33797ebc62b0_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,220 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115409127f696c445a29fb8ca1dc9525b44_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115409127f696c445a29fb8ca1dc9525b44_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,222 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155144717cf9bc41f8a5dc3addf4ab6ff8_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155144717cf9bc41f8a5dc3addf4ab6ff8_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,223 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155543e76b99bb412bb39ece21851848d3_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155543e76b99bb412bb39ece21851848d3_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,224 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115586ce7abd2da4db6b74216080669b20d_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115586ce7abd2da4db6b74216080669b20d_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,226 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155df4233683664b909a0c8067502780a0_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155df4233683664b909a0c8067502780a0_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,227 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115750463dd1dc14ee89f81c71b36e93ddc_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115750463dd1dc14ee89f81c71b36e93ddc_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,229 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111575b034cecd274b4eac400ef4970874f1_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111575b034cecd274b4eac400ef4970874f1_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,230 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157997b811ddb04b2f81e20c1514a96e83_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157997b811ddb04b2f81e20c1514a96e83_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,231 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c044032793b4ee2904f07d64323e3f8_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c044032793b4ee2904f07d64323e3f8_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,232 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157ec8f48e033c449eb35a1651080b8b77_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157ec8f48e033c449eb35a1651080b8b77_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,233 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158ead79a1b50f4ecc9d39fd7d7f902a71_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158ead79a1b50f4ecc9d39fd7d7f902a71_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,235 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111594c0d9711618424bb66bb60ff97603d9_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111594c0d9711618424bb66bb60ff97603d9_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,236 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b1bc2d87afe4ad1827d2ec7786253b4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b1bc2d87afe4ad1827d2ec7786253b4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,238 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ae39b522cc354668ab855fadcac56bab_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ae39b522cc354668ab855fadcac56bab_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,239 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ba600182a12b4e5d9ed71c0161107410_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ba600182a12b4e5d9ed71c0161107410_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,241 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cf206b1937844709b4e13c1a2b5a55d4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cf206b1937844709b4e13c1a2b5a55d4_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,242 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d7b7ea107f5b4ce49e2400d228b9add7_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d7b7ea107f5b4ce49e2400d228b9add7_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,243 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dc580d2cec194a73abc6c433f3003f7f_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dc580d2cec194a73abc6c433f3003f7f_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,245 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e3a9958e0616401fb7c345e0401862cf_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e3a9958e0616401fb7c345e0401862cf_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,246 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115eab9d3ee025440ca943d181b72a1dc00_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115eab9d3ee025440ca943d181b72a1dc00_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,247 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fdcd99b8fe0a4d3ca8e13bdc5c46ad0d_ccd4c1fbc17fcc85146b8fbdf8bdc2ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fdcd99b8fe0a4d3ca8e13bdc5c46ad0d_ccd4c1fbc17fcc85146b8fbdf8bdc2ff 2024-11-15T00:46:41,247 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:46:41,252 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:41,254 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:46:41,256 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:46:41,257 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:41,257 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:46:41,257 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631601257"}]},"ts":"9223372036854775807"} 2024-11-15T00:46:41,259 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:46:41,259 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ccd4c1fbc17fcc85146b8fbdf8bdc2ff, NAME => 'TestAcidGuarantees,,1731631572687.ccd4c1fbc17fcc85146b8fbdf8bdc2ff.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:46:41,259 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:46:41,260 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631601259"}]},"ts":"9223372036854775807"} 2024-11-15T00:46:41,261 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:46:41,262 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:41,263 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 102 msec 2024-11-15T00:46:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T00:46:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:46:41,270 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:41,285 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 253), OpenFileDescriptor=449 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1166 (was 1069) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 15), AvailableMemoryMB=2203 (was 1503) - AvailableMemoryMB LEAK? - 2024-11-15T00:46:41,295 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1166, ProcessCount=11, AvailableMemoryMB=2202 2024-11-15T00:46:41,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:46:41,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:46:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:46:41,299 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:46:41,299 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:41,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-15T00:46:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T00:46:41,300 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:46:41,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742290_1466 (size=1061) 2024-11-15T00:46:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T00:46:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T00:46:41,725 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403 2024-11-15T00:46:41,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742291_1467 (size=53) 2024-11-15T00:46:41,742 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T00:46:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T00:46:42,131 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:46:42,131 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 44c72b983f170fb79c2589a0cd8300ec, disabling compactions & flushes 2024-11-15T00:46:42,131 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:42,131 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:42,131 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. after waiting 0 ms 2024-11-15T00:46:42,131 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:42,131 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:42,131 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 44c72b983f170fb79c2589a0cd8300ec: Waiting for close lock at 1731631602131Disabling compacts and flushes for region at 1731631602131Disabling writes for close at 1731631602131Writing region close event to WAL at 1731631602131Closed at 1731631602131 2024-11-15T00:46:42,132 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:46:42,132 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631602132"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631602132"}]},"ts":"1731631602132"} 2024-11-15T00:46:42,134 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:46:42,135 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:46:42,135 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631602135"}]},"ts":"1731631602135"} 2024-11-15T00:46:42,136 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:46:42,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=44c72b983f170fb79c2589a0cd8300ec, ASSIGN}] 2024-11-15T00:46:42,138 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=44c72b983f170fb79c2589a0cd8300ec, ASSIGN 2024-11-15T00:46:42,138 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=44c72b983f170fb79c2589a0cd8300ec, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,34537,1731631472208; forceNewPlan=false, retain=false 2024-11-15T00:46:42,289 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=44c72b983f170fb79c2589a0cd8300ec, regionState=OPENING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:42,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=44c72b983f170fb79c2589a0cd8300ec, ASSIGN because future has completed 2024-11-15T00:46:42,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:46:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T00:46:42,446 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:42,446 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:46:42,447 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,448 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:46:42,448 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,448 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,449 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,450 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:46:42,450 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c72b983f170fb79c2589a0cd8300ec columnFamilyName A 2024-11-15T00:46:42,450 DEBUG [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:42,451 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] regionserver.HStore(327): Store=44c72b983f170fb79c2589a0cd8300ec/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:46:42,451 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,452 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:46:42,452 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c72b983f170fb79c2589a0cd8300ec columnFamilyName B 2024-11-15T00:46:42,452 DEBUG [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:42,453 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] regionserver.HStore(327): Store=44c72b983f170fb79c2589a0cd8300ec/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:46:42,453 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,454 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:46:42,454 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c72b983f170fb79c2589a0cd8300ec columnFamilyName C 2024-11-15T00:46:42,454 DEBUG [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:46:42,454 INFO [StoreOpener-44c72b983f170fb79c2589a0cd8300ec-1 {}] regionserver.HStore(327): Store=44c72b983f170fb79c2589a0cd8300ec/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:46:42,455 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:42,455 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,455 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,456 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,456 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,456 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,457 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:46:42,458 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,460 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:46:42,460 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened 44c72b983f170fb79c2589a0cd8300ec; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65707890, jitterRate=-0.020876139402389526}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:46:42,460 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:42,461 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for 44c72b983f170fb79c2589a0cd8300ec: Running coprocessor pre-open hook at 1731631602448Writing region info on filesystem at 1731631602448Initializing all the Stores at 1731631602448Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631602449 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631602449Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631602449Cleaning up temporary data from old regions at 1731631602456 (+7 ms)Running coprocessor post-open hooks at 1731631602460 (+4 ms)Region opened successfully at 1731631602461 (+1 ms) 2024-11-15T00:46:42,462 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., pid=110, masterSystemTime=1731631602443 2024-11-15T00:46:42,464 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:42,464 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:42,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=44c72b983f170fb79c2589a0cd8300ec, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:42,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:46:42,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-15T00:46:42,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure 44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 in 175 msec 2024-11-15T00:46:42,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-15T00:46:42,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=44c72b983f170fb79c2589a0cd8300ec, ASSIGN in 331 msec 2024-11-15T00:46:42,470 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:46:42,470 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631602470"}]},"ts":"1731631602470"} 2024-11-15T00:46:42,471 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:46:42,472 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:46:42,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-15T00:46:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-15T00:46:43,450 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:43,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3005a7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,452 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f8ef97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,454 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f8714a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,454 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,457 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:46:43,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,33521,1731631471305 2024-11-15T00:46:43,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e8a0f76 2024-11-15T00:46:43,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:46:43,459 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:46:43,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,461 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c357cf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,462 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19657c05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,464 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b245f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,465 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2140b800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,467 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa0f31c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,469 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bbdb881, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,471 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6469fe0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,472 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55734aa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,474 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@674141d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,475 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c03906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,478 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77592ca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,479 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c00c50f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,481 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f13dcef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38935a78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,485 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492f2c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,486 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a73de36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,488 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@768d2c90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,490 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58324be3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:46:43,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:46:43,492 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:46:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:46:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:46:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@632dfc5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:46:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:46:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:46:43,494 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:46:43,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6046cde3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:46:43,495 DEBUG [Thread-2092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,495 DEBUG [Thread-2109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,495 DEBUG [Thread-2103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:46:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:46:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:46:43,504 DEBUG [Thread-2096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,505 DEBUG [Thread-2094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,505 DEBUG [Thread-2105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,506 DEBUG [Thread-2111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,507 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:43,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,507 DEBUG [Thread-2100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:46:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:46:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:46:43,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,510 DEBUG [Thread-2098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-15T00:46:43,511 DEBUG [Thread-2107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:46:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:46:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:46:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:46:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:46:43,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:46:43,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:46:43,515 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:43,516 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:43,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:43,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T00:46:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:46:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2] 2024-11-15T00:46:43,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:43,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:43,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:43,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:43,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:43,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:43,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/3b2f83b9fe644105a22591e6c36784d9 is 50, key is test_row_0/A:col10/1731631603516/Put/seqid=0 2024-11-15T00:46:43,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742292_1468 (size=12001) 2024-11-15T00:46:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631663594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631663594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631663608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:46:43,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631663610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631663611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:43,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:43,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:43,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:43,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:43,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631663719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631663720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631663720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631663730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631663730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:43,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:43,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:43,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:43,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:43,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:46:43,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631663939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631663939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631663939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631663939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631663939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:43,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:43,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/3b2f83b9fe644105a22591e6c36784d9 2024-11-15T00:46:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:43,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/fd6273e76a0f417bb6a024a9fb842fed is 50, key is test_row_0/B:col10/1731631603516/Put/seqid=0 2024-11-15T00:46:44,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742293_1469 (size=12001) 2024-11-15T00:46:44,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/fd6273e76a0f417bb6a024a9fb842fed 2024-11-15T00:46:44,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/53a2e1d1528a481e82ecb2a1d0d5ce6e is 50, key is test_row_0/C:col10/1731631603516/Put/seqid=0 2024-11-15T00:46:44,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742294_1470 (size=12001) 2024-11-15T00:46:44,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/53a2e1d1528a481e82ecb2a1d0d5ce6e 2024-11-15T00:46:44,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:44,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:44,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:46:44,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/3b2f83b9fe644105a22591e6c36784d9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/3b2f83b9fe644105a22591e6c36784d9 2024-11-15T00:46:44,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/3b2f83b9fe644105a22591e6c36784d9, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T00:46:44,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/fd6273e76a0f417bb6a024a9fb842fed as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fd6273e76a0f417bb6a024a9fb842fed 2024-11-15T00:46:44,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fd6273e76a0f417bb6a024a9fb842fed, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T00:46:44,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/53a2e1d1528a481e82ecb2a1d0d5ce6e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/53a2e1d1528a481e82ecb2a1d0d5ce6e 2024-11-15T00:46:44,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/53a2e1d1528a481e82ecb2a1d0d5ce6e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T00:46:44,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 44c72b983f170fb79c2589a0cd8300ec in 680ms, sequenceid=13, compaction requested=false 2024-11-15T00:46:44,197 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T00:46:44,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:44,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631603563, max timestamp=1731631604271Num uniques 859; 2024-11-15T00:46:44,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:46:44,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631603563, max timestamp=1731631604271Num uniques 859; 2024-11-15T00:46:44,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:46:44,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631603563, max timestamp=1731631604271Num uniques 859; 2024-11-15T00:46:44,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=84.58 KB heapSize=242.95 KB 2024-11-15T00:46:44,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:46:44,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:44,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:44,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:44,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:44,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:44,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:44,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:44,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:44,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/6d7e2729d0f54c1bb5d61bfd5cf9ac95 is 50, key is test_row_0/A:col10/1731631604268/Put/seqid=0 2024-11-15T00:46:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742295_1471 (size=16681) 2024-11-15T00:46:44,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/6d7e2729d0f54c1bb5d61bfd5cf9ac95 2024-11-15T00:46:44,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/7e08e0fc17d3413fa327d80ed37135ae is 50, key is test_row_0/B:col10/1731631604268/Put/seqid=0 2024-11-15T00:46:44,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742296_1472 (size=12001) 2024-11-15T00:46:44,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.50 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/7e08e0fc17d3413fa327d80ed37135ae 2024-11-15T00:46:44,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/8c37464f1ed24f4e9a9d1c21adf1ace9 is 50, key is test_row_0/C:col10/1731631604268/Put/seqid=0 2024-11-15T00:46:44,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631604325, max timestamp=1731631604401Num uniques 859; 2024-11-15T00:46:44,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:46:44,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631604325, max timestamp=1731631604401Num uniques 859; 2024-11-15T00:46:44,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631604325, max timestamp=1731631604401Num uniques 859; 2024-11-15T00:46:44,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:46:44,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:46:44,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742297_1473 (size=12001) 2024-11-15T00:46:44,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:44,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:44,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631664477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631664486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631664490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631664493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631664497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:44,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:44,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631664619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631664619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631664620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631664620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631664631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:46:44,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:44,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:44,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.50 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/8c37464f1ed24f4e9a9d1c21adf1ace9 2024-11-15T00:46:44,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/6d7e2729d0f54c1bb5d61bfd5cf9ac95 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/6d7e2729d0f54c1bb5d61bfd5cf9ac95 2024-11-15T00:46:44,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631664839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631664840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631664840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/6d7e2729d0f54c1bb5d61bfd5cf9ac95, entries=250, sequenceid=38, filesize=16.3 K 2024-11-15T00:46:44,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/7e08e0fc17d3413fa327d80ed37135ae as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/7e08e0fc17d3413fa327d80ed37135ae 2024-11-15T00:46:44,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631664842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:44,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/7e08e0fc17d3413fa327d80ed37135ae, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T00:46:44,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/8c37464f1ed24f4e9a9d1c21adf1ace9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/8c37464f1ed24f4e9a9d1c21adf1ace9 2024-11-15T00:46:44,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/8c37464f1ed24f4e9a9d1c21adf1ace9, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T00:46:44,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~52.88 KB/54150, heapSize ~144.05 KB/147504, currentSize=134.18 KB/137400 for 44c72b983f170fb79c2589a0cd8300ec in 599ms, sequenceid=38, compaction requested=false 2024-11-15T00:46:44,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:44,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:44,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-15T00:46:44,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:44,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:44,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/8be9b3122eb14b09ab634b7eb315c7dd is 50, key is test_row_0/A:col10/1731631604491/Put/seqid=0 2024-11-15T00:46:44,918 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:46:44,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742298_1474 (size=14341) 2024-11-15T00:46:44,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:44,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:44,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:44,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631665010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:45,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:45,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631665147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631665149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631665163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631665163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631665163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:45,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:45,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/8be9b3122eb14b09ab634b7eb315c7dd 2024-11-15T00:46:45,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/f697eca52bd340568528fc39546a71bc is 50, key is test_row_0/B:col10/1731631604491/Put/seqid=0 2024-11-15T00:46:45,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742299_1475 (size=12001) 2024-11-15T00:46:45,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/f697eca52bd340568528fc39546a71bc 2024-11-15T00:46:45,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631665379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/3e022fbb5da147c099611f2eed329cac is 50, key is test_row_0/C:col10/1731631604491/Put/seqid=0 2024-11-15T00:46:45,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:45,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:45,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742300_1476 (size=12001) 2024-11-15T00:46:45,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:46:45,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631665679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631665690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631665690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631665690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631665699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:45,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:45,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:45,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:45,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/3e022fbb5da147c099611f2eed329cac 2024-11-15T00:46:45,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/8be9b3122eb14b09ab634b7eb315c7dd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/8be9b3122eb14b09ab634b7eb315c7dd 2024-11-15T00:46:45,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/8be9b3122eb14b09ab634b7eb315c7dd, entries=200, sequenceid=74, filesize=14.0 K 2024-11-15T00:46:45,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/f697eca52bd340568528fc39546a71bc as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f697eca52bd340568528fc39546a71bc 2024-11-15T00:46:45,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f697eca52bd340568528fc39546a71bc, entries=150, sequenceid=74, filesize=11.7 K 2024-11-15T00:46:45,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/3e022fbb5da147c099611f2eed329cac as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3e022fbb5da147c099611f2eed329cac 2024-11-15T00:46:45,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3e022fbb5da147c099611f2eed329cac, entries=150, sequenceid=74, filesize=11.7 K 2024-11-15T00:46:45,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 44c72b983f170fb79c2589a0cd8300ec in 950ms, sequenceid=74, compaction requested=true 2024-11-15T00:46:45,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:45,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:45,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:45,834 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:45,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:45,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:45,834 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:45,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:45,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:45,834 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:45,834 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:45,834 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:46:45,834 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:46:45,834 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,834 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,835 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/3b2f83b9fe644105a22591e6c36784d9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/6d7e2729d0f54c1bb5d61bfd5cf9ac95, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/8be9b3122eb14b09ab634b7eb315c7dd] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=42.0 K 2024-11-15T00:46:45,835 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fd6273e76a0f417bb6a024a9fb842fed, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/7e08e0fc17d3413fa327d80ed37135ae, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f697eca52bd340568528fc39546a71bc] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=35.2 K 2024-11-15T00:46:45,835 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b2f83b9fe644105a22591e6c36784d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731631603512 2024-11-15T00:46:45,835 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting fd6273e76a0f417bb6a024a9fb842fed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731631603512 2024-11-15T00:46:45,835 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d7e2729d0f54c1bb5d61bfd5cf9ac95, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731631603563 2024-11-15T00:46:45,835 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e08e0fc17d3413fa327d80ed37135ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731631604268 2024-11-15T00:46:45,836 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f697eca52bd340568528fc39546a71bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731631604478 2024-11-15T00:46:45,836 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8be9b3122eb14b09ab634b7eb315c7dd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731631604478 2024-11-15T00:46:45,846 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:45,847 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/d3a0f7577fed49338171e36928f82f16 is 50, key is test_row_0/A:col10/1731631604491/Put/seqid=0 2024-11-15T00:46:45,858 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:45,859 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/395b2443356241d4992e0a7a57c81131 is 50, key is test_row_0/B:col10/1731631604491/Put/seqid=0 2024-11-15T00:46:45,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742301_1477 (size=12104) 2024-11-15T00:46:45,868 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/d3a0f7577fed49338171e36928f82f16 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d3a0f7577fed49338171e36928f82f16 2024-11-15T00:46:45,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T00:46:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,870 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T00:46:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:45,875 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into d3a0f7577fed49338171e36928f82f16(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:45,875 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:45,875 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=13, startTime=1731631605833; duration=0sec 2024-11-15T00:46:45,875 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:45,875 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:46:45,875 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:45,876 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:45,876 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:46:45,876 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:45,876 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/53a2e1d1528a481e82ecb2a1d0d5ce6e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/8c37464f1ed24f4e9a9d1c21adf1ace9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3e022fbb5da147c099611f2eed329cac] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=35.2 K 2024-11-15T00:46:45,877 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53a2e1d1528a481e82ecb2a1d0d5ce6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731631603512 2024-11-15T00:46:45,877 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c37464f1ed24f4e9a9d1c21adf1ace9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731631604268 2024-11-15T00:46:45,878 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e022fbb5da147c099611f2eed329cac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731631604478 2024-11-15T00:46:45,886 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:45,887 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/563eb5b041c14db6bb4edcebb4539c06 is 50, key is test_row_0/C:col10/1731631604491/Put/seqid=0 2024-11-15T00:46:45,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/fb6ebf5444da47fb87dcf46855feaa70 is 50, key is test_row_0/A:col10/1731631604963/Put/seqid=0 2024-11-15T00:46:45,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742302_1478 (size=12104) 2024-11-15T00:46:45,897 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/395b2443356241d4992e0a7a57c81131 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/395b2443356241d4992e0a7a57c81131 2024-11-15T00:46:45,902 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into 395b2443356241d4992e0a7a57c81131(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:45,902 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:45,902 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=13, startTime=1731631605834; duration=0sec 2024-11-15T00:46:45,902 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:45,902 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:46:45,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742303_1479 (size=12104) 2024-11-15T00:46:45,941 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/563eb5b041c14db6bb4edcebb4539c06 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/563eb5b041c14db6bb4edcebb4539c06 2024-11-15T00:46:45,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742304_1480 (size=12001) 2024-11-15T00:46:45,950 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into 563eb5b041c14db6bb4edcebb4539c06(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:45,950 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:45,950 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=13, startTime=1731631605834; duration=0sec 2024-11-15T00:46:45,950 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:45,950 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:46:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:46,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:46,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631606284, max timestamp=1731631606331Num uniques 859; 2024-11-15T00:46:46,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631606284, max timestamp=1731631606331Num uniques 859; 2024-11-15T00:46:46,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:46,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631606284, max timestamp=1731631606331Num uniques 859; 2024-11-15T00:46:46,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:46,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:46,350 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/fb6ebf5444da47fb87dcf46855feaa70 2024-11-15T00:46:46,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/797e9d10b33d4aeaadfc5dc0e85a0bef is 50, key is test_row_0/B:col10/1731631604963/Put/seqid=0 2024-11-15T00:46:46,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742305_1481 (size=12001) 2024-11-15T00:46:46,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631666567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:46,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631666679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:46,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631666700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:46,706 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in 44c72b983f170fb79c2589a0cd8300ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:46,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631666709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:46,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631666710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:46,724 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in 44c72b983f170fb79c2589a0cd8300ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:46,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631666710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:46,725 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in 44c72b983f170fb79c2589a0cd8300ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:46,729 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in 44c72b983f170fb79c2589a0cd8300ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:46:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:46,761 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/797e9d10b33d4aeaadfc5dc0e85a0bef 2024-11-15T00:46:46,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/bcd11b4d556346218eacc4f4478fd812 is 50, key is test_row_0/C:col10/1731631604963/Put/seqid=0 2024-11-15T00:46:46,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742306_1482 (size=12001) 2024-11-15T00:46:46,812 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/bcd11b4d556346218eacc4f4478fd812 2024-11-15T00:46:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/fb6ebf5444da47fb87dcf46855feaa70 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fb6ebf5444da47fb87dcf46855feaa70 2024-11-15T00:46:46,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fb6ebf5444da47fb87dcf46855feaa70, entries=150, sequenceid=83, filesize=11.7 K 2024-11-15T00:46:46,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/797e9d10b33d4aeaadfc5dc0e85a0bef as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/797e9d10b33d4aeaadfc5dc0e85a0bef 2024-11-15T00:46:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631666899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:46,935 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/797e9d10b33d4aeaadfc5dc0e85a0bef, entries=150, sequenceid=83, filesize=11.7 K 2024-11-15T00:46:46,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/bcd11b4d556346218eacc4f4478fd812 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/bcd11b4d556346218eacc4f4478fd812 2024-11-15T00:46:46,957 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/bcd11b4d556346218eacc4f4478fd812, entries=150, sequenceid=83, filesize=11.7 K 2024-11-15T00:46:46,958 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 44c72b983f170fb79c2589a0cd8300ec in 1088ms, sequenceid=83, compaction requested=false 2024-11-15T00:46:46,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:46,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:46,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-15T00:46:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-15T00:46:46,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-15T00:46:46,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4440 sec 2024-11-15T00:46:46,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.4720 sec 2024-11-15T00:46:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:47,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-15T00:46:47,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:47,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T00:46:47,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:47,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T00:46:47,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:47,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T00:46:47,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/2e7470fad5eb42e4b5e672f63b53b544 is 50, key is test_row_0/A:col10/1731631606546/Put/seqid=0 2024-11-15T00:46:47,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742307_1483 (size=14341) 2024-11-15T00:46:47,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631667325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:47,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631667449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:47,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631667659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:47,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/2e7470fad5eb42e4b5e672f63b53b544 2024-11-15T00:46:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:46:47,688 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:47,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:47,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/42b2f97d3b224b5ba888813f854df6cd is 50, key is test_row_0/B:col10/1731631606546/Put/seqid=0 2024-11-15T00:46:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-15T00:46:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T00:46:47,697 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:47,698 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:47,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:47,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742308_1484 (size=12001) 2024-11-15T00:46:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T00:46:47,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T00:46:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:47,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:47,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:47,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:47,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631667979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:48,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T00:46:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:48,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T00:46:48,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/42b2f97d3b224b5ba888813f854df6cd 2024-11-15T00:46:48,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b04f5d743f154096ab7075718e4fcd78 is 50, key is test_row_0/C:col10/1731631606546/Put/seqid=0 2024-11-15T00:46:48,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T00:46:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:48,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742309_1485 (size=12001) 2024-11-15T00:46:48,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b04f5d743f154096ab7075718e4fcd78 2024-11-15T00:46:48,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/2e7470fad5eb42e4b5e672f63b53b544 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/2e7470fad5eb42e4b5e672f63b53b544 2024-11-15T00:46:48,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/2e7470fad5eb42e4b5e672f63b53b544, entries=200, sequenceid=124, filesize=14.0 K 2024-11-15T00:46:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/42b2f97d3b224b5ba888813f854df6cd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/42b2f97d3b224b5ba888813f854df6cd 2024-11-15T00:46:48,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/42b2f97d3b224b5ba888813f854df6cd, entries=150, sequenceid=124, filesize=11.7 K 2024-11-15T00:46:48,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b04f5d743f154096ab7075718e4fcd78 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b04f5d743f154096ab7075718e4fcd78 2024-11-15T00:46:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T00:46:48,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T00:46:48,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:48,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:48,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:48,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:48,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:48,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b04f5d743f154096ab7075718e4fcd78, entries=150, sequenceid=124, filesize=11.7 K 2024-11-15T00:46:48,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 44c72b983f170fb79c2589a0cd8300ec in 1109ms, sequenceid=124, compaction requested=true 2024-11-15T00:46:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:46:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T00:46:48,349 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:48,349 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:48,353 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:48,353 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:46:48,353 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:48,353 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/563eb5b041c14db6bb4edcebb4539c06, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/bcd11b4d556346218eacc4f4478fd812, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b04f5d743f154096ab7075718e4fcd78] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=35.3 K 2024-11-15T00:46:48,353 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:48,353 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:46:48,353 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:48,353 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d3a0f7577fed49338171e36928f82f16, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fb6ebf5444da47fb87dcf46855feaa70, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/2e7470fad5eb42e4b5e672f63b53b544] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=37.5 K 2024-11-15T00:46:48,354 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 563eb5b041c14db6bb4edcebb4539c06, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731631604478 2024-11-15T00:46:48,354 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3a0f7577fed49338171e36928f82f16, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731631604478 2024-11-15T00:46:48,356 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb6ebf5444da47fb87dcf46855feaa70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731631604941 2024-11-15T00:46:48,356 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd11b4d556346218eacc4f4478fd812, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731631604941 2024-11-15T00:46:48,359 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e7470fad5eb42e4b5e672f63b53b544, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731631606488 2024-11-15T00:46:48,359 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting b04f5d743f154096ab7075718e4fcd78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731631606488 2024-11-15T00:46:48,369 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:48,370 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:48,370 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/f24bd1b4a879434ebf64589b1eb46bfe is 50, key is test_row_0/C:col10/1731631606546/Put/seqid=0 2024-11-15T00:46:48,370 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/69e2608a0daf415db2e672e12a0d7f5c is 50, key is test_row_0/A:col10/1731631606546/Put/seqid=0 2024-11-15T00:46:48,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742310_1486 (size=12207) 2024-11-15T00:46:48,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742311_1487 (size=12207) 2024-11-15T00:46:48,392 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/f24bd1b4a879434ebf64589b1eb46bfe as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f24bd1b4a879434ebf64589b1eb46bfe 2024-11-15T00:46:48,398 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/69e2608a0daf415db2e672e12a0d7f5c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/69e2608a0daf415db2e672e12a0d7f5c 2024-11-15T00:46:48,401 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into f24bd1b4a879434ebf64589b1eb46bfe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:48,401 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:48,402 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=13, startTime=1731631608349; duration=0sec 2024-11-15T00:46:48,402 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:48,402 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:46:48,402 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:48,402 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into 69e2608a0daf415db2e672e12a0d7f5c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:48,403 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:48,403 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=13, startTime=1731631608349; duration=0sec 2024-11-15T00:46:48,403 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:48,403 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:46:48,403 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:48,403 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:46:48,403 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:48,403 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/395b2443356241d4992e0a7a57c81131, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/797e9d10b33d4aeaadfc5dc0e85a0bef, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/42b2f97d3b224b5ba888813f854df6cd] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=35.3 K 2024-11-15T00:46:48,403 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 395b2443356241d4992e0a7a57c81131, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731631604478 2024-11-15T00:46:48,404 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 797e9d10b33d4aeaadfc5dc0e85a0bef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731631604941 2024-11-15T00:46:48,404 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b2f97d3b224b5ba888813f854df6cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731631606488 2024-11-15T00:46:48,431 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:48,431 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/c8f79f978812405db0208ae6662c77a9 is 50, key is test_row_0/B:col10/1731631606546/Put/seqid=0 2024-11-15T00:46:48,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742312_1488 (size=12207) 2024-11-15T00:46:48,460 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/c8f79f978812405db0208ae6662c77a9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c8f79f978812405db0208ae6662c77a9 2024-11-15T00:46:48,470 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into c8f79f978812405db0208ae6662c77a9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:48,470 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:48,470 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=13, startTime=1731631608349; duration=0sec 2024-11-15T00:46:48,470 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:48,470 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:46:48,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T00:46:48,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:48,487 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T00:46:48,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:48,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:48,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:48,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:48,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:48,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/fc2ae3c97d034343ac17b574d96a925e is 50, key is test_row_0/A:col10/1731631607266/Put/seqid=0 2024-11-15T00:46:48,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:48,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742313_1489 (size=12101) 2024-11-15T00:46:48,566 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/fc2ae3c97d034343ac17b574d96a925e 2024-11-15T00:46:48,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/099c27874fc24f7b8284449dc16d43e6 is 50, key is test_row_0/B:col10/1731631607266/Put/seqid=0 2024-11-15T00:46:48,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742314_1490 (size=12101) 2024-11-15T00:46:48,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631608680, max timestamp=1731631608749Num uniques 859; 2024-11-15T00:46:48,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631608680, max timestamp=1731631608749Num uniques 859; 2024-11-15T00:46:48,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:48,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:48,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631608680, max timestamp=1731631608749Num uniques 859; 2024-11-15T00:46:48,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:48,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731631608773, max timestamp=1731631608809Num uniques 1068; 2024-11-15T00:46:48,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-15T00:46:48,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731631608773, max timestamp=1731631608809Num uniques 1068; 2024-11-15T00:46:48,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731631608773, max timestamp=1731631608809Num uniques 1068; 2024-11-15T00:46:48,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-15T00:46:48,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-15T00:46:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T00:46:48,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631668881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:48,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631668884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:48,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631668886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631668887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631668893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:48,991 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/099c27874fc24f7b8284449dc16d43e6 2024-11-15T00:46:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/02139429135949a8bf2f0488023a75ff is 50, key is test_row_0/C:col10/1731631607266/Put/seqid=0 2024-11-15T00:46:49,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742315_1491 (size=12101) 2024-11-15T00:46:49,011 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/02139429135949a8bf2f0488023a75ff 2024-11-15T00:46:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/fc2ae3c97d034343ac17b574d96a925e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fc2ae3c97d034343ac17b574d96a925e 2024-11-15T00:46:49,020 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fc2ae3c97d034343ac17b574d96a925e, entries=150, sequenceid=135, filesize=11.8 K 2024-11-15T00:46:49,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/099c27874fc24f7b8284449dc16d43e6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/099c27874fc24f7b8284449dc16d43e6 2024-11-15T00:46:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631669009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631669011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631669011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631669016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631669019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,030 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/099c27874fc24f7b8284449dc16d43e6, entries=150, sequenceid=135, filesize=11.8 K 2024-11-15T00:46:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/02139429135949a8bf2f0488023a75ff as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/02139429135949a8bf2f0488023a75ff 2024-11-15T00:46:49,035 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/02139429135949a8bf2f0488023a75ff, entries=150, sequenceid=135, filesize=11.8 K 2024-11-15T00:46:49,036 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 44c72b983f170fb79c2589a0cd8300ec in 549ms, sequenceid=135, compaction requested=false 2024-11-15T00:46:49,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:49,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:49,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-15T00:46:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-15T00:46:49,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-15T00:46:49,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3400 sec 2024-11-15T00:46:49,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.3520 sec 2024-11-15T00:46:49,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=161.02 KB heapSize=450.47 KB 2024-11-15T00:46:49,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:49,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T00:46:49,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:49,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T00:46:49,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:49,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T00:46:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:49,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/bff037aa8b9b48dc8f5a02c9e9fb7e64 is 50, key is test_row_0/A:col10/1731631609238/Put/seqid=0 2024-11-15T00:46:49,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742316_1492 (size=14541) 2024-11-15T00:46:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631669243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631669244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631669277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631669277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631669277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631669400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631669400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631669400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631669400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631669400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631669622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631669622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631669622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631669622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631669624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/bff037aa8b9b48dc8f5a02c9e9fb7e64 2024-11-15T00:46:49,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/691582fe09ef47ca8b49dc3aa2df5b88 is 50, key is test_row_0/B:col10/1731631609238/Put/seqid=0 2024-11-15T00:46:49,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742317_1493 (size=12151) 2024-11-15T00:46:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T00:46:49,858 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:49,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-15T00:46:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T00:46:49,873 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:49,874 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:49,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:49,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631669940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631669941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631669941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631669941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631669941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:49,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T00:46:50,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T00:46:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/691582fe09ef47ca8b49dc3aa2df5b88 2024-11-15T00:46:50,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/f9c926ab835245eeb816874722ebce58 is 50, key is test_row_0/C:col10/1731631609238/Put/seqid=0 2024-11-15T00:46:50,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742318_1494 (size=12151) 2024-11-15T00:46:50,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T00:46:50,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:50,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T00:46:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T00:46:50,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:50,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631670459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631670459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631670459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631670459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:50,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631670469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:50,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T00:46:50,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:50,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:50,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/f9c926ab835245eeb816874722ebce58 2024-11-15T00:46:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T00:46:50,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/bff037aa8b9b48dc8f5a02c9e9fb7e64 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/bff037aa8b9b48dc8f5a02c9e9fb7e64 2024-11-15T00:46:50,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/bff037aa8b9b48dc8f5a02c9e9fb7e64, entries=200, sequenceid=190, filesize=14.2 K 2024-11-15T00:46:50,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/691582fe09ef47ca8b49dc3aa2df5b88 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/691582fe09ef47ca8b49dc3aa2df5b88 2024-11-15T00:46:50,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/691582fe09ef47ca8b49dc3aa2df5b88, entries=150, sequenceid=190, filesize=11.9 K 2024-11-15T00:46:50,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/f9c926ab835245eeb816874722ebce58 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f9c926ab835245eeb816874722ebce58 2024-11-15T00:46:50,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f9c926ab835245eeb816874722ebce58, entries=150, sequenceid=190, filesize=11.9 K 2024-11-15T00:46:50,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~450.42 KB/461232, currentSize=26.84 KB/27480 for 44c72b983f170fb79c2589a0cd8300ec in 1275ms, sequenceid=190, compaction requested=true 2024-11-15T00:46:50,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:50,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:50,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:50,515 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:50,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:50,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:50,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:50,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:46:50,516 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:50,518 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38849 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:50,518 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:46:50,518 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,518 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/69e2608a0daf415db2e672e12a0d7f5c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fc2ae3c97d034343ac17b574d96a925e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/bff037aa8b9b48dc8f5a02c9e9fb7e64] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=37.9 K 2024-11-15T00:46:50,518 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:50,518 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:46:50,518 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69e2608a0daf415db2e672e12a0d7f5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731631606488 2024-11-15T00:46:50,518 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,518 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c8f79f978812405db0208ae6662c77a9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/099c27874fc24f7b8284449dc16d43e6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/691582fe09ef47ca8b49dc3aa2df5b88] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=35.6 K 2024-11-15T00:46:50,519 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc2ae3c97d034343ac17b574d96a925e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731631607266 2024-11-15T00:46:50,519 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f79f978812405db0208ae6662c77a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731631606488 2024-11-15T00:46:50,519 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting bff037aa8b9b48dc8f5a02c9e9fb7e64, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731631608875 2024-11-15T00:46:50,519 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 099c27874fc24f7b8284449dc16d43e6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731631607266 2024-11-15T00:46:50,519 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 691582fe09ef47ca8b49dc3aa2df5b88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731631608885 2024-11-15T00:46:50,526 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:50,526 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/15052bc6520848bda864b28157f8b3a1 is 50, key is test_row_0/A:col10/1731631609238/Put/seqid=0 2024-11-15T00:46:50,529 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:50,529 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/9742f897a26f4c02b575f192e2a15a05 is 50, key is test_row_0/B:col10/1731631609238/Put/seqid=0 2024-11-15T00:46:50,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742319_1495 (size=12459) 2024-11-15T00:46:50,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742320_1496 (size=12459) 2024-11-15T00:46:50,542 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/9742f897a26f4c02b575f192e2a15a05 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/9742f897a26f4c02b575f192e2a15a05 2024-11-15T00:46:50,545 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into 9742f897a26f4c02b575f192e2a15a05(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:50,546 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:50,546 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=13, startTime=1731631610515; duration=0sec 2024-11-15T00:46:50,546 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:50,546 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:46:50,546 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:50,547 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:50,547 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:46:50,547 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,547 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f24bd1b4a879434ebf64589b1eb46bfe, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/02139429135949a8bf2f0488023a75ff, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f9c926ab835245eeb816874722ebce58] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=35.6 K 2024-11-15T00:46:50,548 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f24bd1b4a879434ebf64589b1eb46bfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731631606488 2024-11-15T00:46:50,548 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 02139429135949a8bf2f0488023a75ff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731631607266 2024-11-15T00:46:50,548 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f9c926ab835245eeb816874722ebce58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731631608885 2024-11-15T00:46:50,553 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:50,554 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/6746b82dc27f4a40b0647caffdc6ded7 is 50, key is test_row_0/C:col10/1731631609238/Put/seqid=0 2024-11-15T00:46:50,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742321_1497 (size=12459) 2024-11-15T00:46:50,575 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/6746b82dc27f4a40b0647caffdc6ded7 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6746b82dc27f4a40b0647caffdc6ded7 2024-11-15T00:46:50,609 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into 6746b82dc27f4a40b0647caffdc6ded7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:50,609 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:50,609 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=13, startTime=1731631610516; duration=0sec 2024-11-15T00:46:50,609 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:50,609 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:46:50,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T00:46:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:50,647 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-15T00:46:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:50,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/17a239f299ce45e384ce9d9b5901b69f is 50, key is test_row_0/A:col10/1731631609247/Put/seqid=0 2024-11-15T00:46:50,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742322_1498 (size=12151) 2024-11-15T00:46:50,936 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/15052bc6520848bda864b28157f8b3a1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/15052bc6520848bda864b28157f8b3a1 2024-11-15T00:46:50,947 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into 15052bc6520848bda864b28157f8b3a1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:50,947 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:50,947 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=13, startTime=1731631610515; duration=0sec 2024-11-15T00:46:50,948 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:50,948 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:46:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T00:46:51,066 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/17a239f299ce45e384ce9d9b5901b69f 2024-11-15T00:46:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/714ff9546f3b45ddae2f42b6c6201435 is 50, key is test_row_0/B:col10/1731631609247/Put/seqid=0 2024-11-15T00:46:51,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742323_1499 (size=12151) 2024-11-15T00:46:51,164 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/714ff9546f3b45ddae2f42b6c6201435 2024-11-15T00:46:51,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/3486a91134b1456aaa3e867606b03287 is 50, key is test_row_0/C:col10/1731631609247/Put/seqid=0 2024-11-15T00:46:51,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742324_1500 (size=12151) 2024-11-15T00:46:51,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/3486a91134b1456aaa3e867606b03287 2024-11-15T00:46:51,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/17a239f299ce45e384ce9d9b5901b69f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/17a239f299ce45e384ce9d9b5901b69f 2024-11-15T00:46:51,325 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/17a239f299ce45e384ce9d9b5901b69f, entries=150, sequenceid=199, filesize=11.9 K 2024-11-15T00:46:51,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/714ff9546f3b45ddae2f42b6c6201435 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/714ff9546f3b45ddae2f42b6c6201435 2024-11-15T00:46:51,345 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/714ff9546f3b45ddae2f42b6c6201435, entries=150, sequenceid=199, filesize=11.9 K 2024-11-15T00:46:51,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/3486a91134b1456aaa3e867606b03287 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3486a91134b1456aaa3e867606b03287 2024-11-15T00:46:51,357 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3486a91134b1456aaa3e867606b03287, entries=150, sequenceid=199, filesize=11.9 K 2024-11-15T00:46:51,361 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 44c72b983f170fb79c2589a0cd8300ec in 714ms, sequenceid=199, compaction requested=false 2024-11-15T00:46:51,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:51,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:51,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-15T00:46:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-15T00:46:51,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-15T00:46:51,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5000 sec 2024-11-15T00:46:51,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.5260 sec 2024-11-15T00:46:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:51,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T00:46:51,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:51,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:51,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:51,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:51,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:51,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:51,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/dbe7ded8f6114427b39ba5f3f34f6ca9 is 50, key is test_row_0/A:col10/1731631611502/Put/seqid=0 2024-11-15T00:46:51,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742325_1501 (size=19321) 2024-11-15T00:46:51,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/dbe7ded8f6114427b39ba5f3f34f6ca9 2024-11-15T00:46:51,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/3b1a75be73df46f39ac26c5f1bb8932f is 50, key is test_row_0/B:col10/1731631611502/Put/seqid=0 2024-11-15T00:46:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631671551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631671554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631611545, max timestamp=1731631611566Num uniques 859; 2024-11-15T00:46:51,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631611545, max timestamp=1731631611566Num uniques 859; 2024-11-15T00:46:51,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:46:51,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:46:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731631611545, max timestamp=1731631611566Num uniques 859; 2024-11-15T00:46:51,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T00:46:51,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742326_1502 (size=12151) 2024-11-15T00:46:51,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/3b1a75be73df46f39ac26c5f1bb8932f 2024-11-15T00:46:51,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/10f8e84075ce42248b2b1b09968fb838 is 50, key is test_row_0/C:col10/1731631611502/Put/seqid=0 2024-11-15T00:46:51,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742327_1503 (size=12151) 2024-11-15T00:46:51,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/10f8e84075ce42248b2b1b09968fb838 2024-11-15T00:46:51,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/dbe7ded8f6114427b39ba5f3f34f6ca9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/dbe7ded8f6114427b39ba5f3f34f6ca9 2024-11-15T00:46:51,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/dbe7ded8f6114427b39ba5f3f34f6ca9, entries=300, sequenceid=211, filesize=18.9 K 2024-11-15T00:46:51,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/3b1a75be73df46f39ac26c5f1bb8932f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3b1a75be73df46f39ac26c5f1bb8932f 2024-11-15T00:46:51,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3b1a75be73df46f39ac26c5f1bb8932f, entries=150, sequenceid=211, filesize=11.9 K 2024-11-15T00:46:51,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/10f8e84075ce42248b2b1b09968fb838 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/10f8e84075ce42248b2b1b09968fb838 2024-11-15T00:46:51,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/10f8e84075ce42248b2b1b09968fb838, entries=150, sequenceid=211, filesize=11.9 K 2024-11-15T00:46:51,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=120.76 KB/123660 for 44c72b983f170fb79c2589a0cd8300ec in 142ms, sequenceid=211, compaction requested=true 2024-11-15T00:46:51,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:51,647 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:51,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:51,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:51,647 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:51,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:51,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:51,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:51,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:51,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-15T00:46:51,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:51,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:51,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:51,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:51,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:51,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:51,650 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:51,650 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:46:51,650 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:51,650 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/9742f897a26f4c02b575f192e2a15a05, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/714ff9546f3b45ddae2f42b6c6201435, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3b1a75be73df46f39ac26c5f1bb8932f] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=35.9 K 2024-11-15T00:46:51,650 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:51,650 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 9742f897a26f4c02b575f192e2a15a05, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731631608885 2024-11-15T00:46:51,650 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:46:51,650 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:51,650 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/15052bc6520848bda864b28157f8b3a1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/17a239f299ce45e384ce9d9b5901b69f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/dbe7ded8f6114427b39ba5f3f34f6ca9] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=42.9 K 2024-11-15T00:46:51,651 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15052bc6520848bda864b28157f8b3a1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731631608885 2024-11-15T00:46:51,651 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 714ff9546f3b45ddae2f42b6c6201435, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731631609240 2024-11-15T00:46:51,652 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17a239f299ce45e384ce9d9b5901b69f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731631609240 2024-11-15T00:46:51,652 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b1a75be73df46f39ac26c5f1bb8932f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731631611500 2024-11-15T00:46:51,653 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbe7ded8f6114427b39ba5f3f34f6ca9, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731631611482 2024-11-15T00:46:51,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/cc911e9e5a6e49cbbbdf5a2f37c088da is 50, key is test_row_0/A:col10/1731631611632/Put/seqid=0 2024-11-15T00:46:51,664 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:51,664 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/e6c45569440d4a2ea2120bf451e76334 is 50, key is test_row_0/B:col10/1731631611502/Put/seqid=0 2024-11-15T00:46:51,667 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:51,668 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/1bde81a675d7446bb720f391aff93a21 is 50, key is test_row_0/A:col10/1731631611502/Put/seqid=0 2024-11-15T00:46:51,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742328_1504 (size=14541) 2024-11-15T00:46:51,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/cc911e9e5a6e49cbbbdf5a2f37c088da 2024-11-15T00:46:51,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742329_1505 (size=12561) 2024-11-15T00:46:51,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742330_1506 (size=12561) 2024-11-15T00:46:51,720 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/1bde81a675d7446bb720f391aff93a21 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1bde81a675d7446bb720f391aff93a21 2024-11-15T00:46:51,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/e0247a9e6c1f404297f10938db06b4f4 is 50, key is test_row_0/B:col10/1731631611632/Put/seqid=0 2024-11-15T00:46:51,738 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into 1bde81a675d7446bb720f391aff93a21(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:51,738 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:51,738 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=13, startTime=1731631611647; duration=0sec 2024-11-15T00:46:51,738 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:51,738 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:46:51,739 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:51,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631671679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631671679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,742 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T00:46:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631671683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,742 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T00:46:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,748 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:51,748 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:46:51,749 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:51,749 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6746b82dc27f4a40b0647caffdc6ded7, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3486a91134b1456aaa3e867606b03287, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/10f8e84075ce42248b2b1b09968fb838] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=35.9 K 2024-11-15T00:46:51,749 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6746b82dc27f4a40b0647caffdc6ded7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731631608885 2024-11-15T00:46:51,750 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3486a91134b1456aaa3e867606b03287, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731631609240 2024-11-15T00:46:51,750 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10f8e84075ce42248b2b1b09968fb838, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731631611500 2024-11-15T00:46:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631671720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631671721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,758 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:51,759 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/594d4ebbe7b94163b0721b13e73d6729 is 50, key is test_row_0/C:col10/1731631611502/Put/seqid=0 2024-11-15T00:46:51,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742331_1507 (size=12151) 2024-11-15T00:46:51,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/e0247a9e6c1f404297f10938db06b4f4 2024-11-15T00:46:51,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742332_1508 (size=12561) 2024-11-15T00:46:51,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/5e72f66d407c4d6b97ac44864b0f4410 is 50, key is test_row_0/C:col10/1731631611632/Put/seqid=0 2024-11-15T00:46:51,833 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/594d4ebbe7b94163b0721b13e73d6729 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/594d4ebbe7b94163b0721b13e73d6729 2024-11-15T00:46:51,841 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into 594d4ebbe7b94163b0721b13e73d6729(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:51,841 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:51,841 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=13, startTime=1731631611647; duration=0sec 2024-11-15T00:46:51,841 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:51,841 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:46:51,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742333_1509 (size=12151) 2024-11-15T00:46:51,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/5e72f66d407c4d6b97ac44864b0f4410 2024-11-15T00:46:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631671849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/cc911e9e5a6e49cbbbdf5a2f37c088da as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/cc911e9e5a6e49cbbbdf5a2f37c088da 2024-11-15T00:46:51,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631671860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631671860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/cc911e9e5a6e49cbbbdf5a2f37c088da, entries=200, sequenceid=245, filesize=14.2 K 2024-11-15T00:46:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/e0247a9e6c1f404297f10938db06b4f4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e0247a9e6c1f404297f10938db06b4f4 2024-11-15T00:46:51,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631671949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631671951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:51,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e0247a9e6c1f404297f10938db06b4f4, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T00:46:51,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/5e72f66d407c4d6b97ac44864b0f4410 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/5e72f66d407c4d6b97ac44864b0f4410 2024-11-15T00:46:52,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/5e72f66d407c4d6b97ac44864b0f4410, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T00:46:52,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 44c72b983f170fb79c2589a0cd8300ec in 373ms, sequenceid=245, compaction requested=false 2024-11-15T00:46:52,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T00:46:52,023 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:52,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-15T00:46:52,026 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:52,027 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:52,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T00:46:52,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:46:52,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:52,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:52,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:52,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:52,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:52,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:52,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/63a5e86f47e748179f389cef9ba7e403 is 50, key is test_row_0/A:col10/1731631611668/Put/seqid=0 2024-11-15T00:46:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T00:46:52,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742334_1510 (size=14591) 2024-11-15T00:46:52,147 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/e6c45569440d4a2ea2120bf451e76334 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e6c45569440d4a2ea2120bf451e76334 2024-11-15T00:46:52,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/63a5e86f47e748179f389cef9ba7e403 2024-11-15T00:46:52,177 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into e6c45569440d4a2ea2120bf451e76334(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:52,177 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:52,177 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=13, startTime=1731631611647; duration=0sec 2024-11-15T00:46:52,177 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:52,177 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:46:52,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/4ec9c3e83b83428aa89754d462980948 is 50, key is test_row_0/B:col10/1731631611668/Put/seqid=0 2024-11-15T00:46:52,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742335_1511 (size=12201) 2024-11-15T00:46:52,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/4ec9c3e83b83428aa89754d462980948 2024-11-15T00:46:52,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/068fa635e81c43f8bdb79c03bd938a82 is 50, key is test_row_0/C:col10/1731631611668/Put/seqid=0 2024-11-15T00:46:52,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742336_1512 (size=12201) 2024-11-15T00:46:52,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/068fa635e81c43f8bdb79c03bd938a82 2024-11-15T00:46:52,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731631612186, max timestamp=1731631612246Num uniques 859; 2024-11-15T00:46:52,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:52,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731631612186, max timestamp=1731631612246Num uniques 859; 2024-11-15T00:46:52,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:52,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731631612186, max timestamp=1731631612246Num uniques 859; 2024-11-15T00:46:52,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:46:52,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/63a5e86f47e748179f389cef9ba7e403 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/63a5e86f47e748179f389cef9ba7e403 2024-11-15T00:46:52,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/63a5e86f47e748179f389cef9ba7e403, entries=200, sequenceid=259, filesize=14.2 K 2024-11-15T00:46:52,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/4ec9c3e83b83428aa89754d462980948 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4ec9c3e83b83428aa89754d462980948 2024-11-15T00:46:52,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4ec9c3e83b83428aa89754d462980948, entries=150, sequenceid=259, filesize=11.9 K 2024-11-15T00:46:52,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/068fa635e81c43f8bdb79c03bd938a82 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/068fa635e81c43f8bdb79c03bd938a82 2024-11-15T00:46:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T00:46:52,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/068fa635e81c43f8bdb79c03bd938a82, entries=150, sequenceid=259, filesize=11.9 K 2024-11-15T00:46:52,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=67.09 KB/68700 for 44c72b983f170fb79c2589a0cd8300ec in 270ms, sequenceid=259, compaction requested=true 2024-11-15T00:46:52,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:52,360 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:52,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:52,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:52,361 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:52,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:52,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:52,361 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:52,361 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:46:52,362 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,362 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1bde81a675d7446bb720f391aff93a21, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/cc911e9e5a6e49cbbbdf5a2f37c088da, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/63a5e86f47e748179f389cef9ba7e403] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=40.7 K 2024-11-15T00:46:52,362 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36913 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:52,362 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:46:52,362 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,362 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e6c45569440d4a2ea2120bf451e76334, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e0247a9e6c1f404297f10938db06b4f4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4ec9c3e83b83428aa89754d462980948] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=36.0 K 2024-11-15T00:46:52,362 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bde81a675d7446bb720f391aff93a21, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731631611500 2024-11-15T00:46:52,362 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c45569440d4a2ea2120bf451e76334, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731631611500 2024-11-15T00:46:52,363 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc911e9e5a6e49cbbbdf5a2f37c088da, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631611592 2024-11-15T00:46:52,363 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e0247a9e6c1f404297f10938db06b4f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631611632 2024-11-15T00:46:52,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:52,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:52,363 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63a5e86f47e748179f389cef9ba7e403, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731631611668 2024-11-15T00:46:52,364 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ec9c3e83b83428aa89754d462980948, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731631611668 2024-11-15T00:46:52,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=73.80 KB heapSize=201 KB 2024-11-15T00:46:52,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:52,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:52,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:52,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:52,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:52,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:52,393 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:52,394 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/fe69281047c94425b3a62947b6965003 is 50, key is test_row_0/B:col10/1731631611668/Put/seqid=0 2024-11-15T00:46:52,397 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:52,397 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/5a2ad7dc92524598a9a5a07f186a9d1e is 50, key is test_row_0/A:col10/1731631611668/Put/seqid=0 2024-11-15T00:46:52,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/ac82b27e70eb4ff69baac41068a1c414 is 50, key is test_row_0/A:col10/1731631612379/Put/seqid=0 2024-11-15T00:46:52,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631672417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631672429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631672439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631672439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631672439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742338_1514 (size=12713) 2024-11-15T00:46:52,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742339_1515 (size=17181) 2024-11-15T00:46:52,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742337_1513 (size=12713) 2024-11-15T00:46:52,519 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/5a2ad7dc92524598a9a5a07f186a9d1e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5a2ad7dc92524598a9a5a07f186a9d1e 2024-11-15T00:46:52,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:52,532 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into 5a2ad7dc92524598a9a5a07f186a9d1e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:52,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,532 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:52,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:52,532 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=13, startTime=1731631612360; duration=0sec 2024-11-15T00:46:52,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,532 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:52,532 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:46:52,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,532 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:52,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,537 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/fe69281047c94425b3a62947b6965003 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fe69281047c94425b3a62947b6965003 2024-11-15T00:46:52,537 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36913 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:52,538 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:46:52,538 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,538 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/594d4ebbe7b94163b0721b13e73d6729, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/5e72f66d407c4d6b97ac44864b0f4410, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/068fa635e81c43f8bdb79c03bd938a82] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=36.0 K 2024-11-15T00:46:52,538 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 594d4ebbe7b94163b0721b13e73d6729, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731631611500 2024-11-15T00:46:52,539 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e72f66d407c4d6b97ac44864b0f4410, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631611632 2024-11-15T00:46:52,540 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 068fa635e81c43f8bdb79c03bd938a82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731631611668 2024-11-15T00:46:52,544 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into fe69281047c94425b3a62947b6965003(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:52,544 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:52,544 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=13, startTime=1731631612361; duration=0sec 2024-11-15T00:46:52,544 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:52,544 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:46:52,554 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:52,554 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/d88f2a1bd51142c984739a4563e06f57 is 50, key is test_row_0/C:col10/1731631611668/Put/seqid=0 2024-11-15T00:46:52,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631672549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631672551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631672570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631672570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742340_1516 (size=12713) 2024-11-15T00:46:52,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631672576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T00:46:52,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:52,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:52,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631672770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631672780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631672790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631672792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631672800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:52,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:52,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:52,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/ac82b27e70eb4ff69baac41068a1c414 2024-11-15T00:46:52,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/db8234b0abf04b7fa78e8187cf81e5cd is 50, key is test_row_0/B:col10/1731631612379/Put/seqid=0 2024-11-15T00:46:52,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742341_1517 (size=12301) 2024-11-15T00:46:52,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/db8234b0abf04b7fa78e8187cf81e5cd 2024-11-15T00:46:53,000 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/d88f2a1bd51142c984739a4563e06f57 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/d88f2a1bd51142c984739a4563e06f57 2024-11-15T00:46:53,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/be6989162633492a9dbf11839d1d738a is 50, key is test_row_0/C:col10/1731631612379/Put/seqid=0 2024-11-15T00:46:53,013 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into d88f2a1bd51142c984739a4563e06f57(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:53,013 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:53,013 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=13, startTime=1731631612361; duration=0sec 2024-11-15T00:46:53,013 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:53,013 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:46:53,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:53,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:53,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742342_1518 (size=12301) 2024-11-15T00:46:53,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/be6989162633492a9dbf11839d1d738a 2024-11-15T00:46:53,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/ac82b27e70eb4ff69baac41068a1c414 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ac82b27e70eb4ff69baac41068a1c414 2024-11-15T00:46:53,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ac82b27e70eb4ff69baac41068a1c414, entries=250, sequenceid=292, filesize=16.8 K 2024-11-15T00:46:53,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/db8234b0abf04b7fa78e8187cf81e5cd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/db8234b0abf04b7fa78e8187cf81e5cd 2024-11-15T00:46:53,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/db8234b0abf04b7fa78e8187cf81e5cd, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T00:46:53,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/be6989162633492a9dbf11839d1d738a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/be6989162633492a9dbf11839d1d738a 2024-11-15T00:46:53,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/be6989162633492a9dbf11839d1d738a, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T00:46:53,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~300.56 KB/307776, currentSize=73.80 KB/75570 for 44c72b983f170fb79c2589a0cd8300ec in 687ms, sequenceid=292, compaction requested=false 2024-11-15T00:46:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:53,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T00:46:53,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:53,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:53,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:53,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:53,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:53,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:53,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/4bf566d26f8940eb93af1b1a950f2a25 is 50, key is test_row_0/A:col10/1731631613106/Put/seqid=0 2024-11-15T00:46:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T00:46:53,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:53,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:53,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631673180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631673180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742343_1519 (size=12301) 2024-11-15T00:46:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/4bf566d26f8940eb93af1b1a950f2a25 2024-11-15T00:46:53,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631673210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631673212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631673212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/867c9f9436b448a5938790abcd990583 is 50, key is test_row_0/B:col10/1731631613106/Put/seqid=0 2024-11-15T00:46:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631673319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631673319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:53,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:53,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742344_1520 (size=12301) 2024-11-15T00:46:53,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631673340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631673340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631673340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:53,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:53,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631673539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631673539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631673569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631673569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631673570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/867c9f9436b448a5938790abcd990583 2024-11-15T00:46:53,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/72df85411ca144a8a80b0c562b1af6a2 is 50, key is test_row_0/C:col10/1731631613106/Put/seqid=0 2024-11-15T00:46:53,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742345_1521 (size=12301) 2024-11-15T00:46:53,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/72df85411ca144a8a80b0c562b1af6a2 2024-11-15T00:46:53,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/4bf566d26f8940eb93af1b1a950f2a25 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/4bf566d26f8940eb93af1b1a950f2a25 2024-11-15T00:46:53,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/4bf566d26f8940eb93af1b1a950f2a25, entries=150, sequenceid=311, filesize=12.0 K 2024-11-15T00:46:53,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/867c9f9436b448a5938790abcd990583 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/867c9f9436b448a5938790abcd990583 2024-11-15T00:46:53,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/867c9f9436b448a5938790abcd990583, entries=150, sequenceid=311, filesize=12.0 K 2024-11-15T00:46:53,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/72df85411ca144a8a80b0c562b1af6a2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/72df85411ca144a8a80b0c562b1af6a2 2024-11-15T00:46:53,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/72df85411ca144a8a80b0c562b1af6a2, entries=150, sequenceid=311, filesize=12.0 K 2024-11-15T00:46:53,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 44c72b983f170fb79c2589a0cd8300ec in 732ms, sequenceid=311, compaction requested=true 2024-11-15T00:46:53,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:53,843 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:53,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:53,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:53,844 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:53,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:53,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:53,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:53,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:53,851 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:53,851 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:46:53,851 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,851 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5a2ad7dc92524598a9a5a07f186a9d1e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ac82b27e70eb4ff69baac41068a1c414, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/4bf566d26f8940eb93af1b1a950f2a25] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=41.2 K 2024-11-15T00:46:53,851 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:53,852 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:46:53,852 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,852 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fe69281047c94425b3a62947b6965003, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/db8234b0abf04b7fa78e8187cf81e5cd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/867c9f9436b448a5938790abcd990583] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=36.4 K 2024-11-15T00:46:53,852 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a2ad7dc92524598a9a5a07f186a9d1e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731631611668 2024-11-15T00:46:53,854 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting fe69281047c94425b3a62947b6965003, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731631611668 2024-11-15T00:46:53,854 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac82b27e70eb4ff69baac41068a1c414, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731631612246 2024-11-15T00:46:53,855 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting db8234b0abf04b7fa78e8187cf81e5cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731631612363 2024-11-15T00:46:53,855 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf566d26f8940eb93af1b1a950f2a25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731631612435 2024-11-15T00:46:53,855 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 867c9f9436b448a5938790abcd990583, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731631612435 2024-11-15T00:46:53,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T00:46:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:53,874 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:53,874 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/47bec14a7f074f1088074e360592fcf1 is 50, key is test_row_0/B:col10/1731631613106/Put/seqid=0 2024-11-15T00:46:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:53,886 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:53,886 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/ec11f8f21b2f4d338104da38f815a4ff is 50, key is test_row_0/A:col10/1731631613106/Put/seqid=0 2024-11-15T00:46:53,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/c1381988923e49ba8cfab965ca449534 is 50, key is test_row_0/A:col10/1731631613868/Put/seqid=0 2024-11-15T00:46:53,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742346_1522 (size=12915) 2024-11-15T00:46:53,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742348_1524 (size=17181) 2024-11-15T00:46:53,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/c1381988923e49ba8cfab965ca449534 2024-11-15T00:46:53,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742347_1523 (size=12915) 2024-11-15T00:46:53,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/5b84e5f94a594dccbb47f8cfcf4bf29a is 50, key is test_row_0/B:col10/1731631613868/Put/seqid=0 2024-11-15T00:46:53,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742349_1525 (size=12301) 2024-11-15T00:46:53,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/5b84e5f94a594dccbb47f8cfcf4bf29a 2024-11-15T00:46:53,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/9a76ef34c14c43ab8df55029f97f0958 is 50, key is test_row_0/C:col10/1731631613868/Put/seqid=0 2024-11-15T00:46:53,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742350_1526 (size=12301) 2024-11-15T00:46:53,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:53,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:53,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:53,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:53,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631673949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631673949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631673951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631673973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:53,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631673977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631674079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631674079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631674079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631674090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631674101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:54,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:54,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:54,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:54,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T00:46:54,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:54,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:54,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631674296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631674298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631674299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,314 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/47bec14a7f074f1088074e360592fcf1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/47bec14a7f074f1088074e360592fcf1 2024-11-15T00:46:54,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631674310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631674310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,321 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into 47bec14a7f074f1088074e360592fcf1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:54,321 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:54,321 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=13, startTime=1731631613844; duration=0sec 2024-11-15T00:46:54,322 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:54,322 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:46:54,322 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:54,323 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:54,323 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:46:54,323 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:54,324 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/d88f2a1bd51142c984739a4563e06f57, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/be6989162633492a9dbf11839d1d738a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/72df85411ca144a8a80b0c562b1af6a2] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=36.4 K 2024-11-15T00:46:54,327 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/ec11f8f21b2f4d338104da38f815a4ff as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ec11f8f21b2f4d338104da38f815a4ff 2024-11-15T00:46:54,332 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting d88f2a1bd51142c984739a4563e06f57, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731631611668 2024-11-15T00:46:54,332 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into ec11f8f21b2f4d338104da38f815a4ff(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:54,332 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:54,332 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=13, startTime=1731631613843; duration=0sec 2024-11-15T00:46:54,332 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:54,332 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:46:54,333 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting be6989162633492a9dbf11839d1d738a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731631612363 2024-11-15T00:46:54,333 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 72df85411ca144a8a80b0c562b1af6a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731631612435 2024-11-15T00:46:54,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/9a76ef34c14c43ab8df55029f97f0958 2024-11-15T00:46:54,358 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:54,359 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/4727ec47c0634391bb397270c226df05 is 50, key is test_row_0/C:col10/1731631613106/Put/seqid=0 2024-11-15T00:46:54,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/c1381988923e49ba8cfab965ca449534 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/c1381988923e49ba8cfab965ca449534 2024-11-15T00:46:54,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/c1381988923e49ba8cfab965ca449534, entries=250, sequenceid=330, filesize=16.8 K 2024-11-15T00:46:54,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/5b84e5f94a594dccbb47f8cfcf4bf29a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5b84e5f94a594dccbb47f8cfcf4bf29a 2024-11-15T00:46:54,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5b84e5f94a594dccbb47f8cfcf4bf29a, entries=150, sequenceid=330, filesize=12.0 K 2024-11-15T00:46:54,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/9a76ef34c14c43ab8df55029f97f0958 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9a76ef34c14c43ab8df55029f97f0958 2024-11-15T00:46:54,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9a76ef34c14c43ab8df55029f97f0958, entries=150, sequenceid=330, filesize=12.0 K 2024-11-15T00:46:54,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 44c72b983f170fb79c2589a0cd8300ec in 521ms, sequenceid=330, compaction requested=false 2024-11-15T00:46:54,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:54,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742351_1527 (size=12915) 2024-11-15T00:46:54,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T00:46:54,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:54,440 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:46:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:54,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/d2121678bbf54b3da5af177bbe00d1f2 is 50, key is test_row_0/A:col10/1731631613969/Put/seqid=0 2024-11-15T00:46:54,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742352_1528 (size=12301) 2024-11-15T00:46:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:54,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631674641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631674643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631674642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631674651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631674654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631674759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631674759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631674760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631674760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631674769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,828 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/4727ec47c0634391bb397270c226df05 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/4727ec47c0634391bb397270c226df05 2024-11-15T00:46:54,851 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into 4727ec47c0634391bb397270c226df05(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:54,851 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:54,851 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=13, startTime=1731631613844; duration=0sec 2024-11-15T00:46:54,851 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:54,851 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:46:54,920 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/d2121678bbf54b3da5af177bbe00d1f2 2024-11-15T00:46:54,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/d0108e48aadd427db711e007588e42e4 is 50, key is test_row_0/B:col10/1731631613969/Put/seqid=0 2024-11-15T00:46:54,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742353_1529 (size=12301) 2024-11-15T00:46:54,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631674980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631674980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631674980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631674980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631674980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:55,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631675300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:55,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631675300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:55,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631675304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:55,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631675310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:55,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:55,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631675314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:55,348 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/d0108e48aadd427db711e007588e42e4 2024-11-15T00:46:55,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/acac18f00f16457f92bc90ef3d7e58a0 is 50, key is test_row_0/C:col10/1731631613969/Put/seqid=0 2024-11-15T00:46:55,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742354_1530 (size=12301) 2024-11-15T00:46:55,435 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/acac18f00f16457f92bc90ef3d7e58a0 2024-11-15T00:46:55,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/d2121678bbf54b3da5af177bbe00d1f2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d2121678bbf54b3da5af177bbe00d1f2 2024-11-15T00:46:55,457 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d2121678bbf54b3da5af177bbe00d1f2, entries=150, sequenceid=345, filesize=12.0 K 2024-11-15T00:46:55,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/d0108e48aadd427db711e007588e42e4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/d0108e48aadd427db711e007588e42e4 2024-11-15T00:46:55,463 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/d0108e48aadd427db711e007588e42e4, entries=150, sequenceid=345, filesize=12.0 K 2024-11-15T00:46:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/acac18f00f16457f92bc90ef3d7e58a0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/acac18f00f16457f92bc90ef3d7e58a0 2024-11-15T00:46:55,493 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/acac18f00f16457f92bc90ef3d7e58a0, entries=150, sequenceid=345, filesize=12.0 K 2024-11-15T00:46:55,496 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 44c72b983f170fb79c2589a0cd8300ec in 1056ms, sequenceid=345, compaction requested=true 2024-11-15T00:46:55,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:55,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:55,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-15T00:46:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-15T00:46:55,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-15T00:46:55,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4810 sec 2024-11-15T00:46:55,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 3.4950 sec 2024-11-15T00:46:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:55,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:46:55,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:55,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/d7325a73518f43d5becb54c7b6bd292a is 50, key is test_row_0/A:col10/1731631614650/Put/seqid=0 2024-11-15T00:46:55,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742355_1531 (size=14741) 2024-11-15T00:46:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631675852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631675851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:55,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631675900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:55,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631675900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:55,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631675900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631676009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631676009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631676019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631676019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631676030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T00:46:56,199 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:56,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-15T00:46:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:46:56,201 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:56,202 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:56,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631676230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631676230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631676240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631676248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631676250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/d7325a73518f43d5becb54c7b6bd292a 2024-11-15T00:46:56,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/596ab698b3844962a4c8eb355c04aad7 is 50, key is test_row_0/B:col10/1731631614650/Put/seqid=0 2024-11-15T00:46:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:46:56,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742356_1532 (size=12301) 2024-11-15T00:46:56,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:46:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:56,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:56,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:46:56,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:56,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:56,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:56,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:56,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:46:56,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631676550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631676559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631676559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631676578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631676578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:56,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:46:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:56,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:56,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/596ab698b3844962a4c8eb355c04aad7 2024-11-15T00:46:56,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b66b7dd44d5841ce891a1bc5b1d5fccf is 50, key is test_row_0/C:col10/1731631614650/Put/seqid=0 2024-11-15T00:46:56,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742357_1533 (size=12301) 2024-11-15T00:46:56,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b66b7dd44d5841ce891a1bc5b1d5fccf 2024-11-15T00:46:56,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/d7325a73518f43d5becb54c7b6bd292a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d7325a73518f43d5becb54c7b6bd292a 2024-11-15T00:46:56,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d7325a73518f43d5becb54c7b6bd292a, entries=200, sequenceid=366, filesize=14.4 K 2024-11-15T00:46:56,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/596ab698b3844962a4c8eb355c04aad7 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/596ab698b3844962a4c8eb355c04aad7 2024-11-15T00:46:56,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/596ab698b3844962a4c8eb355c04aad7, entries=150, sequenceid=366, filesize=12.0 K 2024-11-15T00:46:56,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b66b7dd44d5841ce891a1bc5b1d5fccf as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b66b7dd44d5841ce891a1bc5b1d5fccf 2024-11-15T00:46:56,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b66b7dd44d5841ce891a1bc5b1d5fccf, entries=150, sequenceid=366, filesize=12.0 K 2024-11-15T00:46:56,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 44c72b983f170fb79c2589a0cd8300ec in 972ms, sequenceid=366, compaction requested=true 2024-11-15T00:46:56,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:56,800 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:56,800 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:56,801 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:56,801 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:46:56,801 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:56,801 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/47bec14a7f074f1088074e360592fcf1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5b84e5f94a594dccbb47f8cfcf4bf29a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/d0108e48aadd427db711e007588e42e4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/596ab698b3844962a4c8eb355c04aad7] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=48.7 K 2024-11-15T00:46:56,802 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57138 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:56,802 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:46:56,802 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:56,802 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 47bec14a7f074f1088074e360592fcf1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731631612435 2024-11-15T00:46:56,802 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ec11f8f21b2f4d338104da38f815a4ff, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/c1381988923e49ba8cfab965ca449534, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d2121678bbf54b3da5af177bbe00d1f2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d7325a73518f43d5becb54c7b6bd292a] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=55.8 K 2024-11-15T00:46:56,803 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec11f8f21b2f4d338104da38f815a4ff, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731631612435 2024-11-15T00:46:56,803 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b84e5f94a594dccbb47f8cfcf4bf29a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731631613209 2024-11-15T00:46:56,803 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting d0108e48aadd427db711e007588e42e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731631613947 2024-11-15T00:46:56,803 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1381988923e49ba8cfab965ca449534, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731631613170 2024-11-15T00:46:56,803 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2121678bbf54b3da5af177bbe00d1f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731631613947 2024-11-15T00:46:56,803 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 596ab698b3844962a4c8eb355c04aad7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731631614642 2024-11-15T00:46:56,803 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7325a73518f43d5becb54c7b6bd292a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731631614642 2024-11-15T00:46:56,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:46:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:56,814 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:46:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:56,815 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:56,815 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/b7aa6fc1d14b4c7fba23921a3cf5b0bf is 50, key is test_row_0/A:col10/1731631614650/Put/seqid=0 2024-11-15T00:46:56,819 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:56,819 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/4f361492da1b4f5fa44e8a4307656bee is 50, key is test_row_0/B:col10/1731631614650/Put/seqid=0 2024-11-15T00:46:56,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/1de0fdaca7454372811c7b831732563b is 50, key is test_row_0/A:col10/1731631615863/Put/seqid=0 2024-11-15T00:46:56,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742359_1535 (size=13051) 2024-11-15T00:46:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:46:56,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742358_1534 (size=13051) 2024-11-15T00:46:56,834 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/4f361492da1b4f5fa44e8a4307656bee as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4f361492da1b4f5fa44e8a4307656bee 2024-11-15T00:46:56,838 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/b7aa6fc1d14b4c7fba23921a3cf5b0bf as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b7aa6fc1d14b4c7fba23921a3cf5b0bf 2024-11-15T00:46:56,844 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into b7aa6fc1d14b4c7fba23921a3cf5b0bf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:56,844 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into 4f361492da1b4f5fa44e8a4307656bee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:56,844 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:56,844 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:56,844 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=12, startTime=1731631616800; duration=0sec 2024-11-15T00:46:56,844 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=12, startTime=1731631616800; duration=0sec 2024-11-15T00:46:56,844 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:56,844 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:56,844 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:46:56,844 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:46:56,844 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:46:56,845 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:46:56,846 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:46:56,846 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:56,846 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/4727ec47c0634391bb397270c226df05, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9a76ef34c14c43ab8df55029f97f0958, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/acac18f00f16457f92bc90ef3d7e58a0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b66b7dd44d5841ce891a1bc5b1d5fccf] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=48.7 K 2024-11-15T00:46:56,846 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 4727ec47c0634391bb397270c226df05, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731631612435 2024-11-15T00:46:56,846 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a76ef34c14c43ab8df55029f97f0958, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731631613209 2024-11-15T00:46:56,847 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting acac18f00f16457f92bc90ef3d7e58a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731631613947 2024-11-15T00:46:56,847 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting b66b7dd44d5841ce891a1bc5b1d5fccf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731631614642 2024-11-15T00:46:56,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742360_1536 (size=12301) 2024-11-15T00:46:56,857 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:56,857 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/ce65316e7475481db0b082e7d18b8a19 is 50, key is test_row_0/C:col10/1731631614650/Put/seqid=0 2024-11-15T00:46:56,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742361_1537 (size=13051) 2024-11-15T00:46:56,869 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/ce65316e7475481db0b082e7d18b8a19 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/ce65316e7475481db0b082e7d18b8a19 2024-11-15T00:46:56,876 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into ce65316e7475481db0b082e7d18b8a19(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:56,876 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:56,876 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=12, startTime=1731631616800; duration=0sec 2024-11-15T00:46:56,876 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:56,876 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:46:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:57,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:57,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631677112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631677113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631677114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631677120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631677122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631677229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631677229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631677230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631677230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631677239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/1de0fdaca7454372811c7b831732563b 2024-11-15T00:46:57,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/c2ed8c4e92a647eda2bbbe2ff7b179b0 is 50, key is test_row_0/B:col10/1731631615863/Put/seqid=0 2024-11-15T00:46:57,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742362_1538 (size=12301) 2024-11-15T00:46:57,278 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/c2ed8c4e92a647eda2bbbe2ff7b179b0 2024-11-15T00:46:57,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/04a83eb50a5c4561b450b26a655ea15e is 50, key is test_row_0/C:col10/1731631615863/Put/seqid=0 2024-11-15T00:46:57,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742363_1539 (size=12301) 2024-11-15T00:46:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:46:57,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631677439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631677450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631677450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631677452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631677460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,708 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/04a83eb50a5c4561b450b26a655ea15e 2024-11-15T00:46:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/1de0fdaca7454372811c7b831732563b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1de0fdaca7454372811c7b831732563b 2024-11-15T00:46:57,717 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1de0fdaca7454372811c7b831732563b, entries=150, sequenceid=378, filesize=12.0 K 2024-11-15T00:46:57,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/c2ed8c4e92a647eda2bbbe2ff7b179b0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c2ed8c4e92a647eda2bbbe2ff7b179b0 2024-11-15T00:46:57,722 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c2ed8c4e92a647eda2bbbe2ff7b179b0, entries=150, sequenceid=378, filesize=12.0 K 2024-11-15T00:46:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/04a83eb50a5c4561b450b26a655ea15e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/04a83eb50a5c4561b450b26a655ea15e 2024-11-15T00:46:57,727 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/04a83eb50a5c4561b450b26a655ea15e, entries=150, sequenceid=378, filesize=12.0 K 2024-11-15T00:46:57,728 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 44c72b983f170fb79c2589a0cd8300ec in 914ms, sequenceid=378, compaction requested=false 2024-11-15T00:46:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-15T00:46:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-15T00:46:57,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-15T00:46:57,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5280 sec 2024-11-15T00:46:57,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.5330 sec 2024-11-15T00:46:57,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631617113, max timestamp=1731631617767Num uniques 859; 2024-11-15T00:46:57,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:46:57,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-15T00:46:57,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:57,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:57,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:57,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:57,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:57,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:46:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:46:57,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/584c830f99494b45acd3bf5dbddb8970 is 50, key is test_row_0/A:col10/1731631617118/Put/seqid=0 2024-11-15T00:46:57,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742364_1540 (size=14741) 2024-11-15T00:46:57,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631677833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631677833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631677838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631677842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631677853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631677959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631677959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631677959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631677969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631677970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631678180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:58,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631678180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631678180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631678190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:58,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:58,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631678190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/584c830f99494b45acd3bf5dbddb8970 2024-11-15T00:46:58,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/f745a6ee2c7447aaa71fdeed29afc547 is 50, key is test_row_0/B:col10/1731631617118/Put/seqid=0 2024-11-15T00:46:58,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742365_1541 (size=12301) 2024-11-15T00:46:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:46:58,351 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:46:58,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:46:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-15T00:46:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:46:58,360 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:46:58,361 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:46:58,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:46:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:46:58,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631678500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:58,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631678500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:58,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:58,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631678500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T00:46:58,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631678509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:58,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:58,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:58,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:58,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631678512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/f745a6ee2c7447aaa71fdeed29afc547 2024-11-15T00:46:58,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/6a50d8475f124e8b999efe5d813f3370 is 50, key is test_row_0/C:col10/1731631617118/Put/seqid=0 2024-11-15T00:46:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:46:58,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T00:46:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:58,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742366_1542 (size=12301) 2024-11-15T00:46:58,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T00:46:58,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:58,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:58,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:58,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:46:58,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T00:46:58,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:58,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:58,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:58,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:59,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631679022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:59,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631679022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:59,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631679022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:59,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631679030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:59,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:46:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631679041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:46:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:46:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:46:59,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/6a50d8475f124e8b999efe5d813f3370 2024-11-15T00:46:59,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T00:46:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:46:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:59,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:59,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:59,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:46:59,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/584c830f99494b45acd3bf5dbddb8970 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/584c830f99494b45acd3bf5dbddb8970 2024-11-15T00:46:59,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/584c830f99494b45acd3bf5dbddb8970, entries=200, sequenceid=402, filesize=14.4 K 2024-11-15T00:46:59,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/f745a6ee2c7447aaa71fdeed29afc547 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f745a6ee2c7447aaa71fdeed29afc547 2024-11-15T00:46:59,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f745a6ee2c7447aaa71fdeed29afc547, entries=150, sequenceid=402, filesize=12.0 K 2024-11-15T00:46:59,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/6a50d8475f124e8b999efe5d813f3370 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6a50d8475f124e8b999efe5d813f3370 2024-11-15T00:46:59,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6a50d8475f124e8b999efe5d813f3370, entries=150, sequenceid=402, filesize=12.0 K 2024-11-15T00:46:59,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 44c72b983f170fb79c2589a0cd8300ec in 1506ms, sequenceid=402, compaction requested=true 2024-11-15T00:46:59,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:59,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:46:59,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:59,279 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:59,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:46:59,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:59,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:46:59,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:46:59,279 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:59,280 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:46:59,280 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:59,280 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b7aa6fc1d14b4c7fba23921a3cf5b0bf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1de0fdaca7454372811c7b831732563b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/584c830f99494b45acd3bf5dbddb8970] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=39.2 K 2024-11-15T00:46:59,280 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:59,280 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7aa6fc1d14b4c7fba23921a3cf5b0bf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731631614642 2024-11-15T00:46:59,280 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1de0fdaca7454372811c7b831732563b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731631615853 2024-11-15T00:46:59,281 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 584c830f99494b45acd3bf5dbddb8970, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731631617113 2024-11-15T00:46:59,281 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:59,281 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:46:59,281 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:59,281 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4f361492da1b4f5fa44e8a4307656bee, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c2ed8c4e92a647eda2bbbe2ff7b179b0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f745a6ee2c7447aaa71fdeed29afc547] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=36.8 K 2024-11-15T00:46:59,282 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f361492da1b4f5fa44e8a4307656bee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731631614642 2024-11-15T00:46:59,282 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c2ed8c4e92a647eda2bbbe2ff7b179b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731631615853 2024-11-15T00:46:59,283 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f745a6ee2c7447aaa71fdeed29afc547, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731631617118 2024-11-15T00:46:59,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T00:46:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:59,309 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T00:46:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:46:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:46:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:46:59,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:46:59,337 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:59,337 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/5e6701aaa8ec429380da0660ec9a7ac9 is 50, key is test_row_0/A:col10/1731631617118/Put/seqid=0 2024-11-15T00:46:59,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/ab555517644847df98e9da98ec7038e2 is 50, key is test_row_0/A:col10/1731631617838/Put/seqid=0 2024-11-15T00:46:59,342 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:59,342 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/8c91b7f54c814b00af38b082c561be31 is 50, key is test_row_0/B:col10/1731631617118/Put/seqid=0 2024-11-15T00:46:59,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742367_1543 (size=13153) 2024-11-15T00:46:59,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742368_1544 (size=12301) 2024-11-15T00:46:59,413 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/ab555517644847df98e9da98ec7038e2 2024-11-15T00:46:59,419 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/5e6701aaa8ec429380da0660ec9a7ac9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5e6701aaa8ec429380da0660ec9a7ac9 2024-11-15T00:46:59,424 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into 5e6701aaa8ec429380da0660ec9a7ac9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:59,425 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:59,425 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=13, startTime=1731631619278; duration=0sec 2024-11-15T00:46:59,425 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:46:59,425 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:46:59,425 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:46:59,434 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:46:59,434 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:46:59,434 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:46:59,434 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/ce65316e7475481db0b082e7d18b8a19, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/04a83eb50a5c4561b450b26a655ea15e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6a50d8475f124e8b999efe5d813f3370] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=36.8 K 2024-11-15T00:46:59,435 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce65316e7475481db0b082e7d18b8a19, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731631614642 2024-11-15T00:46:59,435 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04a83eb50a5c4561b450b26a655ea15e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731631615853 2024-11-15T00:46:59,435 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a50d8475f124e8b999efe5d813f3370, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731631617118 2024-11-15T00:46:59,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742369_1545 (size=13153) 2024-11-15T00:46:59,450 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/8c91b7f54c814b00af38b082c561be31 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/8c91b7f54c814b00af38b082c561be31 2024-11-15T00:46:59,455 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into 8c91b7f54c814b00af38b082c561be31(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:59,456 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:59,456 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=13, startTime=1731631619279; duration=0sec 2024-11-15T00:46:59,456 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:59,456 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:46:59,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/5dc79f8f9fc54b66b1baf86a46efe7cd is 50, key is test_row_0/B:col10/1731631617838/Put/seqid=0 2024-11-15T00:46:59,480 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:46:59,481 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/abf97a304ab64e10a449c0d43714ef1a is 50, key is test_row_0/C:col10/1731631617118/Put/seqid=0 2024-11-15T00:46:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:46:59,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742371_1547 (size=13153) 2024-11-15T00:46:59,555 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/abf97a304ab64e10a449c0d43714ef1a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/abf97a304ab64e10a449c0d43714ef1a 2024-11-15T00:46:59,564 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into abf97a304ab64e10a449c0d43714ef1a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:46:59,564 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:46:59,564 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=13, startTime=1731631619279; duration=0sec 2024-11-15T00:46:59,564 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:46:59,564 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:46:59,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742370_1546 (size=12301) 2024-11-15T00:46:59,988 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/5dc79f8f9fc54b66b1baf86a46efe7cd 2024-11-15T00:47:00,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/cdf5e3df9b2e4d3d8d4934646c055f10 is 50, key is test_row_0/C:col10/1731631617838/Put/seqid=0 2024-11-15T00:47:00,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742372_1548 (size=12301) 2024-11-15T00:47:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:00,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:47:00,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631680159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631680161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631680163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631680164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631680183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631680289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631680300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631680300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631680309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631680309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,332 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:47:00,468 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/cdf5e3df9b2e4d3d8d4934646c055f10 2024-11-15T00:47:00,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/ab555517644847df98e9da98ec7038e2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ab555517644847df98e9da98ec7038e2 2024-11-15T00:47:00,499 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ab555517644847df98e9da98ec7038e2, entries=150, sequenceid=418, filesize=12.0 K 2024-11-15T00:47:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/5dc79f8f9fc54b66b1baf86a46efe7cd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5dc79f8f9fc54b66b1baf86a46efe7cd 2024-11-15T00:47:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:47:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631680512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631680519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631680520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,532 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5dc79f8f9fc54b66b1baf86a46efe7cd, entries=150, sequenceid=418, filesize=12.0 K 2024-11-15T00:47:00,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/cdf5e3df9b2e4d3d8d4934646c055f10 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cdf5e3df9b2e4d3d8d4934646c055f10 2024-11-15T00:47:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631680529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631680533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,561 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cdf5e3df9b2e4d3d8d4934646c055f10, entries=150, sequenceid=418, filesize=12.0 K 2024-11-15T00:47:00,562 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 44c72b983f170fb79c2589a0cd8300ec in 1253ms, sequenceid=418, compaction requested=false 2024-11-15T00:47:00,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:00,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:00,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-15T00:47:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-15T00:47:00,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-15T00:47:00,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2030 sec 2024-11-15T00:47:00,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.2290 sec 2024-11-15T00:47:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:00,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T00:47:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:47:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:47:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:47:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:00,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/b6ab0fe9a97d48cd8f49c82114d49aef is 50, key is test_row_0/A:col10/1731631620174/Put/seqid=0 2024-11-15T00:47:00,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742373_1549 (size=14741) 2024-11-15T00:47:00,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/b6ab0fe9a97d48cd8f49c82114d49aef 2024-11-15T00:47:00,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631680868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/3bc67725f6514890bcf865ddb6f339d2 is 50, key is test_row_0/B:col10/1731631620174/Put/seqid=0 2024-11-15T00:47:00,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631680875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631680880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631680882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742374_1550 (size=12301) 2024-11-15T00:47:00,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631680879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:00,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/3bc67725f6514890bcf865ddb6f339d2 2024-11-15T00:47:00,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b70a4a60b9c4445eb624a0b9dfa4ddba is 50, key is test_row_0/C:col10/1731631620174/Put/seqid=0 2024-11-15T00:47:00,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742375_1551 (size=12301) 2024-11-15T00:47:00,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b70a4a60b9c4445eb624a0b9dfa4ddba 2024-11-15T00:47:00,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/b6ab0fe9a97d48cd8f49c82114d49aef as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b6ab0fe9a97d48cd8f49c82114d49aef 2024-11-15T00:47:00,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b6ab0fe9a97d48cd8f49c82114d49aef, entries=200, sequenceid=438, filesize=14.4 K 2024-11-15T00:47:00,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/3bc67725f6514890bcf865ddb6f339d2 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3bc67725f6514890bcf865ddb6f339d2 2024-11-15T00:47:00,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3bc67725f6514890bcf865ddb6f339d2, entries=150, sequenceid=438, filesize=12.0 K 2024-11-15T00:47:00,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b70a4a60b9c4445eb624a0b9dfa4ddba as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b70a4a60b9c4445eb624a0b9dfa4ddba 2024-11-15T00:47:00,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b70a4a60b9c4445eb624a0b9dfa4ddba, entries=150, sequenceid=438, filesize=12.0 K 2024-11-15T00:47:00,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 44c72b983f170fb79c2589a0cd8300ec in 159ms, sequenceid=438, compaction requested=true 2024-11-15T00:47:00,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:00,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:00,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:00,991 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:00,991 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:00,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:47:00,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:00,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:47:00,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:00,992 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:00,992 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:00,992 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:47:00,992 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:47:00,992 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:00,992 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:00,992 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/8c91b7f54c814b00af38b082c561be31, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5dc79f8f9fc54b66b1baf86a46efe7cd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3bc67725f6514890bcf865ddb6f339d2] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=36.9 K 2024-11-15T00:47:00,992 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5e6701aaa8ec429380da0660ec9a7ac9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ab555517644847df98e9da98ec7038e2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b6ab0fe9a97d48cd8f49c82114d49aef] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=39.3 K 2024-11-15T00:47:00,993 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e6701aaa8ec429380da0660ec9a7ac9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731631617118 2024-11-15T00:47:00,993 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c91b7f54c814b00af38b082c561be31, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731631617118 2024-11-15T00:47:00,993 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dc79f8f9fc54b66b1baf86a46efe7cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731631617835 2024-11-15T00:47:00,993 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab555517644847df98e9da98ec7038e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731631617835 2024-11-15T00:47:00,993 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc67725f6514890bcf865ddb6f339d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731631620160 2024-11-15T00:47:00,993 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6ab0fe9a97d48cd8f49c82114d49aef, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731631620160 2024-11-15T00:47:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:01,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T00:47:01,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:47:01,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:01,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:47:01,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:01,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:47:01,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:01,014 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:01,014 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/811c8f5799f44444af258a4783c4e322 is 50, key is test_row_0/A:col10/1731631620174/Put/seqid=0 2024-11-15T00:47:01,018 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:01,018 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/c44ae27570034a899c4e93b176fca72e is 50, key is test_row_0/B:col10/1731631620174/Put/seqid=0 2024-11-15T00:47:01,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/baaa5388cec44705be4cf6890eebb80d is 50, key is test_row_0/A:col10/1731631620854/Put/seqid=0 2024-11-15T00:47:01,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631681035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631681037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631681060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742377_1553 (size=14741) 2024-11-15T00:47:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631681063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631681064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742376_1552 (size=13255) 2024-11-15T00:47:01,111 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/811c8f5799f44444af258a4783c4e322 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/811c8f5799f44444af258a4783c4e322 2024-11-15T00:47:01,115 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into 811c8f5799f44444af258a4783c4e322(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:01,115 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:01,115 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=13, startTime=1731631620991; duration=0sec 2024-11-15T00:47:01,115 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:01,115 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:47:01,115 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:01,116 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:01,116 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:47:01,116 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:01,116 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/abf97a304ab64e10a449c0d43714ef1a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cdf5e3df9b2e4d3d8d4934646c055f10, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b70a4a60b9c4445eb624a0b9dfa4ddba] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=36.9 K 2024-11-15T00:47:01,117 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting abf97a304ab64e10a449c0d43714ef1a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731631617118 2024-11-15T00:47:01,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742378_1554 (size=13255) 2024-11-15T00:47:01,118 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdf5e3df9b2e4d3d8d4934646c055f10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731631617835 2024-11-15T00:47:01,118 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b70a4a60b9c4445eb624a0b9dfa4ddba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731631620160 2024-11-15T00:47:01,127 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:01,127 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/986b112432c746d99f1655c60a067c29 is 50, key is test_row_0/C:col10/1731631620174/Put/seqid=0 2024-11-15T00:47:01,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742379_1555 (size=13255) 2024-11-15T00:47:01,154 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/986b112432c746d99f1655c60a067c29 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/986b112432c746d99f1655c60a067c29 2024-11-15T00:47:01,169 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into 986b112432c746d99f1655c60a067c29(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:01,169 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:01,169 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=13, startTime=1731631620992; duration=0sec 2024-11-15T00:47:01,169 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:01,169 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:47:01,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631681176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631681176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631681189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631681199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631681200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631681390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631681392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631681409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631681419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631681419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/baaa5388cec44705be4cf6890eebb80d 2024-11-15T00:47:01,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/44041b5af20648aa9ecabf3bdd65c5ae is 50, key is test_row_0/B:col10/1731631620854/Put/seqid=0 2024-11-15T00:47:01,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742380_1556 (size=12301) 2024-11-15T00:47:01,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/44041b5af20648aa9ecabf3bdd65c5ae 2024-11-15T00:47:01,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/9d959466c4cb49e4818358c206ff5d4b is 50, key is test_row_0/C:col10/1731631620854/Put/seqid=0 2024-11-15T00:47:01,523 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/c44ae27570034a899c4e93b176fca72e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c44ae27570034a899c4e93b176fca72e 2024-11-15T00:47:01,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742381_1557 (size=12301) 2024-11-15T00:47:01,529 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into c44ae27570034a899c4e93b176fca72e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:01,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/9d959466c4cb49e4818358c206ff5d4b 2024-11-15T00:47:01,529 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:01,529 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=13, startTime=1731631620991; duration=0sec 2024-11-15T00:47:01,529 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:01,529 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:47:01,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/baaa5388cec44705be4cf6890eebb80d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/baaa5388cec44705be4cf6890eebb80d 2024-11-15T00:47:01,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/baaa5388cec44705be4cf6890eebb80d, entries=200, sequenceid=454, filesize=14.4 K 2024-11-15T00:47:01,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/44041b5af20648aa9ecabf3bdd65c5ae as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/44041b5af20648aa9ecabf3bdd65c5ae 2024-11-15T00:47:01,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/44041b5af20648aa9ecabf3bdd65c5ae, entries=150, sequenceid=454, filesize=12.0 K 2024-11-15T00:47:01,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/9d959466c4cb49e4818358c206ff5d4b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9d959466c4cb49e4818358c206ff5d4b 2024-11-15T00:47:01,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9d959466c4cb49e4818358c206ff5d4b, entries=150, sequenceid=454, filesize=12.0 K 2024-11-15T00:47:01,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 44c72b983f170fb79c2589a0cd8300ec in 568ms, sequenceid=454, compaction requested=false 2024-11-15T00:47:01,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:01,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T00:47:01,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:47:01,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:01,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:47:01,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:01,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:47:01,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:01,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/38f26e5cc96d4b238659a7b44cc04197 is 50, key is test_row_0/A:col10/1731631621060/Put/seqid=0 2024-11-15T00:47:01,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742382_1558 (size=14741) 2024-11-15T00:47:01,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631681755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631681757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631681758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631681759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631681761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631681879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631681879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631681879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631681879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631681879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631682099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631682110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:02,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631682110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631682110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631682110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/38f26e5cc96d4b238659a7b44cc04197 2024-11-15T00:47:02,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/819fd8254c594f289f9d377f41b7f672 is 50, key is test_row_0/B:col10/1731631621060/Put/seqid=0 2024-11-15T00:47:02,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742383_1559 (size=12301) 2024-11-15T00:47:02,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631682420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:02,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631682434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631682435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:02,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631682435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:02,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631682436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:47:02,520 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:02,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:47:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-15T00:47:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:47:02,528 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:47:02,529 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:47:02,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:47:02,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/819fd8254c594f289f9d377f41b7f672 2024-11-15T00:47:02,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b29071fd80714be1b82be5daa963e851 is 50, key is test_row_0/C:col10/1731631621060/Put/seqid=0 2024-11-15T00:47:02,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742384_1560 (size=12301) 2024-11-15T00:47:02,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b29071fd80714be1b82be5daa963e851 2024-11-15T00:47:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:47:02,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/38f26e5cc96d4b238659a7b44cc04197 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/38f26e5cc96d4b238659a7b44cc04197 2024-11-15T00:47:02,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/38f26e5cc96d4b238659a7b44cc04197, entries=200, sequenceid=474, filesize=14.4 K 2024-11-15T00:47:02,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/819fd8254c594f289f9d377f41b7f672 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/819fd8254c594f289f9d377f41b7f672 2024-11-15T00:47:02,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/819fd8254c594f289f9d377f41b7f672, entries=150, sequenceid=474, filesize=12.0 K 2024-11-15T00:47:02,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/b29071fd80714be1b82be5daa963e851 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b29071fd80714be1b82be5daa963e851 2024-11-15T00:47:02,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b29071fd80714be1b82be5daa963e851, entries=150, sequenceid=474, filesize=12.0 K 2024-11-15T00:47:02,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T00:47:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:47:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:02,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:02,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 44c72b983f170fb79c2589a0cd8300ec in 974ms, sequenceid=474, compaction requested=true 2024-11-15T00:47:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:02,681 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:02,682 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:02,682 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:47:02,682 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:02,682 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/811c8f5799f44444af258a4783c4e322, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/baaa5388cec44705be4cf6890eebb80d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/38f26e5cc96d4b238659a7b44cc04197] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=41.7 K 2024-11-15T00:47:02,683 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 811c8f5799f44444af258a4783c4e322, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731631620160 2024-11-15T00:47:02,683 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting baaa5388cec44705be4cf6890eebb80d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731631620854 2024-11-15T00:47:02,684 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f26e5cc96d4b238659a7b44cc04197, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731631621057 2024-11-15T00:47:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:47:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:47:02,688 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:02,691 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:02,691 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:47:02,691 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:02,691 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c44ae27570034a899c4e93b176fca72e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/44041b5af20648aa9ecabf3bdd65c5ae, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/819fd8254c594f289f9d377f41b7f672] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=37.0 K 2024-11-15T00:47:02,691 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:02,692 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/5bc64c3a35524d44b2cdcf1fc7537ff0 is 50, key is test_row_0/A:col10/1731631621060/Put/seqid=0 2024-11-15T00:47:02,692 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c44ae27570034a899c4e93b176fca72e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731631620160 2024-11-15T00:47:02,693 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 44041b5af20648aa9ecabf3bdd65c5ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731631620854 2024-11-15T00:47:02,693 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 819fd8254c594f289f9d377f41b7f672, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731631621060 2024-11-15T00:47:02,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742385_1561 (size=13357) 2024-11-15T00:47:02,742 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:02,743 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/04640945aa8246cbaf31ee9159467fe4 is 50, key is test_row_0/B:col10/1731631621060/Put/seqid=0 2024-11-15T00:47:02,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742386_1562 (size=13357) 2024-11-15T00:47:02,777 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/04640945aa8246cbaf31ee9159467fe4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/04640945aa8246cbaf31ee9159467fe4 2024-11-15T00:47:02,785 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into 04640945aa8246cbaf31ee9159467fe4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:02,785 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:02,785 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=13, startTime=1731631622688; duration=0sec 2024-11-15T00:47:02,786 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:02,786 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:47:02,786 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:02,798 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:02,799 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:47:02,799 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:02,799 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/986b112432c746d99f1655c60a067c29, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9d959466c4cb49e4818358c206ff5d4b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b29071fd80714be1b82be5daa963e851] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=37.0 K 2024-11-15T00:47:02,799 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 986b112432c746d99f1655c60a067c29, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731631620160 2024-11-15T00:47:02,799 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d959466c4cb49e4818358c206ff5d4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731631620854 2024-11-15T00:47:02,802 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting b29071fd80714be1b82be5daa963e851, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731631621060 2024-11-15T00:47:02,826 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:02,827 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/cb59c7a967614859a0cb7fe0d1c494a3 is 50, key is test_row_0/C:col10/1731631621060/Put/seqid=0 2024-11-15T00:47:02,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T00:47:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:02,835 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T00:47:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:47:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:47:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:47:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:02,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742387_1563 (size=13357) 2024-11-15T00:47:02,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/af121e39c3464d76b9ab0635ce7c431e is 50, key is test_row_0/A:col10/1731631621757/Put/seqid=0 2024-11-15T00:47:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:47:02,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742388_1564 (size=12301) 2024-11-15T00:47:02,879 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/af121e39c3464d76b9ab0635ce7c431e 2024-11-15T00:47:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/ffb9710a3cc44491b58e5ee7dfeb7055 is 50, key is test_row_0/B:col10/1731631621757/Put/seqid=0 2024-11-15T00:47:02,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742389_1565 (size=12301) 2024-11-15T00:47:02,940 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/ffb9710a3cc44491b58e5ee7dfeb7055 2024-11-15T00:47:02,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. as already flushing 2024-11-15T00:47:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:02,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/24ef4c4f503445e0bae1c36e745a7e96 is 50, key is test_row_0/C:col10/1731631621757/Put/seqid=0 2024-11-15T00:47:03,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631682991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742390_1566 (size=12301) 2024-11-15T00:47:03,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631683008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631683012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631683017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631683016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,049 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/24ef4c4f503445e0bae1c36e745a7e96 2024-11-15T00:47:03,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/af121e39c3464d76b9ab0635ce7c431e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/af121e39c3464d76b9ab0635ce7c431e 2024-11-15T00:47:03,112 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/af121e39c3464d76b9ab0635ce7c431e, entries=150, sequenceid=490, filesize=12.0 K 2024-11-15T00:47:03,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/ffb9710a3cc44491b58e5ee7dfeb7055 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/ffb9710a3cc44491b58e5ee7dfeb7055 2024-11-15T00:47:03,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631683129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,139 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/5bc64c3a35524d44b2cdcf1fc7537ff0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5bc64c3a35524d44b2cdcf1fc7537ff0 2024-11-15T00:47:03,140 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/ffb9710a3cc44491b58e5ee7dfeb7055, entries=150, sequenceid=490, filesize=12.0 K 2024-11-15T00:47:03,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/24ef4c4f503445e0bae1c36e745a7e96 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/24ef4c4f503445e0bae1c36e745a7e96 2024-11-15T00:47:03,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631683150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631683152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631683152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:47:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631683152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,163 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into 5bc64c3a35524d44b2cdcf1fc7537ff0(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:03,163 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:03,163 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=13, startTime=1731631622681; duration=0sec 2024-11-15T00:47:03,163 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:03,163 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:47:03,164 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/24ef4c4f503445e0bae1c36e745a7e96, entries=150, sequenceid=490, filesize=12.0 K 2024-11-15T00:47:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,165 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 44c72b983f170fb79c2589a0cd8300ec in 330ms, sequenceid=490, compaction requested=false 2024-11-15T00:47:03,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:03,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:03,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-15T00:47:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-15T00:47:03,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-15T00:47:03,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 639 msec 2024-11-15T00:47:03,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 651 msec 2024-11-15T00:47:03,257 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/cb59c7a967614859a0cb7fe0d1c494a3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cb59c7a967614859a0cb7fe0d1c494a3 2024-11-15T00:47:03,263 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into cb59c7a967614859a0cb7fe0d1c494a3(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:03,263 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:03,263 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=13, startTime=1731631622688; duration=0sec 2024-11-15T00:47:03,263 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:03,263 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:47:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:03,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T00:47:03,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:47:03,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:03,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:47:03,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:03,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:47:03,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:03,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/0760c5166af744ed8781892953ddecee is 50, key is test_row_0/A:col10/1731631623011/Put/seqid=0 2024-11-15T00:47:03,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742391_1567 (size=14741) 2024-11-15T00:47:03,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631683406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631683418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631683423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631683423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631683428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,515 INFO [Thread-2105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2105. 2024-11-15T00:47:03,515 DEBUG [Thread-2105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:03,515 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:03,515 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:03,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:03,517 INFO [Thread-2109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2109. 2024-11-15T00:47:03,518 DEBUG [Thread-2109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:03,518 DEBUG [Thread-2109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:03,518 DEBUG [Thread-2109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:03,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:03,519 INFO [Thread-2103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2103. 2024-11-15T00:47:03,519 DEBUG [Thread-2103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:03,520 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:03,520 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:03,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:03,531 INFO [Thread-2107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2107. 2024-11-15T00:47:03,531 DEBUG [Thread-2107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:03,531 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:03,531 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:03,531 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:03,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631683531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,535 INFO [Thread-2111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2111. 2024-11-15T00:47:03,535 DEBUG [Thread-2111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:03,535 DEBUG [Thread-2111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:03,535 DEBUG [Thread-2111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:03,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:03,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631683539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631683549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631683549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631683564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:47:03,672 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:03,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631683739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631683752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631683759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/0760c5166af744ed8781892953ddecee 2024-11-15T00:47:03,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631683772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/b3c30af6f9a1481cb157b722c158e9fb is 50, key is test_row_0/B:col10/1731631623011/Put/seqid=0 2024-11-15T00:47:03,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631683780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:03,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742392_1568 (size=12301) 2024-11-15T00:47:04,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631684056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:04,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631684070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:04,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631684059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631684079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:04,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631684092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:04,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/b3c30af6f9a1481cb157b722c158e9fb 2024-11-15T00:47:04,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/2546bc770e294b85a39a0cd01762410b is 50, key is test_row_0/C:col10/1731631623011/Put/seqid=0 2024-11-15T00:47:04,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742393_1569 (size=12301) 2024-11-15T00:47:04,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33976 deadline: 1731631684570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:04,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:04,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34008 deadline: 1731631684579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:04,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:04,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34016 deadline: 1731631684580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:04,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34036 deadline: 1731631684590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:04,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33956 deadline: 1731631684610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:47:04,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/2546bc770e294b85a39a0cd01762410b 2024-11-15T00:47:04,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/0760c5166af744ed8781892953ddecee as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/0760c5166af744ed8781892953ddecee 2024-11-15T00:47:04,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/0760c5166af744ed8781892953ddecee, entries=200, sequenceid=510, filesize=14.4 K 2024-11-15T00:47:04,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/b3c30af6f9a1481cb157b722c158e9fb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/b3c30af6f9a1481cb157b722c158e9fb 2024-11-15T00:47:04,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/b3c30af6f9a1481cb157b722c158e9fb, entries=150, sequenceid=510, filesize=12.0 K 2024-11-15T00:47:04,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/2546bc770e294b85a39a0cd01762410b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/2546bc770e294b85a39a0cd01762410b 2024-11-15T00:47:04,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/2546bc770e294b85a39a0cd01762410b, entries=150, sequenceid=510, filesize=12.0 K 2024-11-15T00:47:04,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 44c72b983f170fb79c2589a0cd8300ec in 1371ms, sequenceid=510, compaction requested=true 2024-11-15T00:47:04,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:04,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:04,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:04,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:47:04,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:47:04,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 44c72b983f170fb79c2589a0cd8300ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:47:04,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T00:47:04,718 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:04,718 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:04,724 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:04,724 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/A is initiating minor compaction (all files) 2024-11-15T00:47:04,724 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/A in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:04,725 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5bc64c3a35524d44b2cdcf1fc7537ff0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/af121e39c3464d76b9ab0635ce7c431e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/0760c5166af744ed8781892953ddecee] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=39.5 K 2024-11-15T00:47:04,725 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:04,725 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/C is initiating minor compaction (all files) 2024-11-15T00:47:04,725 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/C in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:04,725 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cb59c7a967614859a0cb7fe0d1c494a3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/24ef4c4f503445e0bae1c36e745a7e96, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/2546bc770e294b85a39a0cd01762410b] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=37.1 K 2024-11-15T00:47:04,725 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bc64c3a35524d44b2cdcf1fc7537ff0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731631621060 2024-11-15T00:47:04,726 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting cb59c7a967614859a0cb7fe0d1c494a3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731631621060 2024-11-15T00:47:04,726 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting af121e39c3464d76b9ab0635ce7c431e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731631621735 2024-11-15T00:47:04,726 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ef4c4f503445e0bae1c36e745a7e96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731631621735 2024-11-15T00:47:04,726 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0760c5166af744ed8781892953ddecee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731631623006 2024-11-15T00:47:04,728 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2546bc770e294b85a39a0cd01762410b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731631623011 2024-11-15T00:47:04,746 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#C#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:04,747 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/c5fa43fbed714f779341e080db1be9ba is 50, key is test_row_0/C:col10/1731631623011/Put/seqid=0 2024-11-15T00:47:04,750 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#A#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:04,751 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/08e9941d8abf47318ff99d8b1a84aba3 is 50, key is test_row_0/A:col10/1731631623011/Put/seqid=0 2024-11-15T00:47:04,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742395_1571 (size=13459) 2024-11-15T00:47:04,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742394_1570 (size=13459) 2024-11-15T00:47:04,821 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/08e9941d8abf47318ff99d8b1a84aba3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/08e9941d8abf47318ff99d8b1a84aba3 2024-11-15T00:47:04,839 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/c5fa43fbed714f779341e080db1be9ba as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/c5fa43fbed714f779341e080db1be9ba 2024-11-15T00:47:04,865 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/A of 44c72b983f170fb79c2589a0cd8300ec into 08e9941d8abf47318ff99d8b1a84aba3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:04,865 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:04,866 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/A, priority=13, startTime=1731631624717; duration=0sec 2024-11-15T00:47:04,866 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:04,866 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:A 2024-11-15T00:47:04,866 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:04,879 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:04,879 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 44c72b983f170fb79c2589a0cd8300ec/B is initiating minor compaction (all files) 2024-11-15T00:47:04,879 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 44c72b983f170fb79c2589a0cd8300ec/B in TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:04,879 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/04640945aa8246cbaf31ee9159467fe4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/ffb9710a3cc44491b58e5ee7dfeb7055, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/b3c30af6f9a1481cb157b722c158e9fb] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp, totalSize=37.1 K 2024-11-15T00:47:04,880 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/C of 44c72b983f170fb79c2589a0cd8300ec into c5fa43fbed714f779341e080db1be9ba(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:04,880 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:04,880 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/C, priority=13, startTime=1731631624717; duration=0sec 2024-11-15T00:47:04,880 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04640945aa8246cbaf31ee9159467fe4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731631621060 2024-11-15T00:47:04,880 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:04,880 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:C 2024-11-15T00:47:04,881 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffb9710a3cc44491b58e5ee7dfeb7055, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731631621735 2024-11-15T00:47:04,886 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3c30af6f9a1481cb157b722c158e9fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731631623011 2024-11-15T00:47:04,906 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 44c72b983f170fb79c2589a0cd8300ec#B#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:04,907 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/3f49bbbc5cae4a78a6cd77ce1296db80 is 50, key is test_row_0/B:col10/1731631623011/Put/seqid=0 2024-11-15T00:47:04,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742396_1572 (size=13459) 2024-11-15T00:47:04,948 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/3f49bbbc5cae4a78a6cd77ce1296db80 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3f49bbbc5cae4a78a6cd77ce1296db80 2024-11-15T00:47:04,954 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 44c72b983f170fb79c2589a0cd8300ec/B of 44c72b983f170fb79c2589a0cd8300ec into 3f49bbbc5cae4a78a6cd77ce1296db80(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:04,954 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:04,954 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., storeName=44c72b983f170fb79c2589a0cd8300ec/B, priority=13, startTime=1731631624717; duration=0sec 2024-11-15T00:47:04,955 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:04,955 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 44c72b983f170fb79c2589a0cd8300ec:B 2024-11-15T00:47:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:05,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T00:47:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:47:05,581 INFO [Thread-2096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2096. 2024-11-15T00:47:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:47:05,581 DEBUG [Thread-2096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:05,581 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:47:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:05,582 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:05,592 INFO [Thread-2094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2094. 2024-11-15T00:47:05,592 DEBUG [Thread-2094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:05,592 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,592 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:05,593 INFO [Thread-2100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2100. 2024-11-15T00:47:05,593 DEBUG [Thread-2100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:05,593 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/73ce752b758746dd9610e30de0fd3522 is 50, key is test_row_0/A:col10/1731631623415/Put/seqid=0 2024-11-15T00:47:05,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:05,593 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742397_1573 (size=12301) 2024-11-15T00:47:05,605 INFO [Thread-2098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2098. 2024-11-15T00:47:05,605 DEBUG [Thread-2098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:05,605 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,605 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:05,621 INFO [Thread-2092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2092. 2024-11-15T00:47:05,622 DEBUG [Thread-2092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:05,622 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,622 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 879 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2628 rows 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 863 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2589 rows 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 871 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2613 rows 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 878 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2625 rows 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 864 2024-11-15T00:47:05,622 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2592 rows 2024-11-15T00:47:05,622 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:47:05,622 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:47:05,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:05,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:05,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:47:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:47:05,627 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631625627"}]},"ts":"1731631625627"} 2024-11-15T00:47:05,628 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:47:05,628 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:47:05,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:47:05,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=44c72b983f170fb79c2589a0cd8300ec, UNASSIGN}] 2024-11-15T00:47:05,632 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=44c72b983f170fb79c2589a0cd8300ec, UNASSIGN 2024-11-15T00:47:05,632 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=44c72b983f170fb79c2589a0cd8300ec, regionState=CLOSING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:05,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=44c72b983f170fb79c2589a0cd8300ec, UNASSIGN because future has completed 2024-11-15T00:47:05,635 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:47:05,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:47:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:47:05,788 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(122): Close 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:05,788 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:47:05,788 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1722): Closing 44c72b983f170fb79c2589a0cd8300ec, disabling compactions & flushes 2024-11-15T00:47:05,788 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:47:06,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/73ce752b758746dd9610e30de0fd3522 2024-11-15T00:47:06,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/e02d983a3424402f8164667864ef703a is 50, key is test_row_0/B:col10/1731631623415/Put/seqid=0 2024-11-15T00:47:06,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742398_1574 (size=12301) 2024-11-15T00:47:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:47:06,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/e02d983a3424402f8164667864ef703a 2024-11-15T00:47:06,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/e7622390bac748edb2619206b6dc1dde is 50, key is test_row_0/C:col10/1731631623415/Put/seqid=0 2024-11-15T00:47:06,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742399_1575 (size=12301) 2024-11-15T00:47:06,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/e7622390bac748edb2619206b6dc1dde 2024-11-15T00:47:06,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/73ce752b758746dd9610e30de0fd3522 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/73ce752b758746dd9610e30de0fd3522 2024-11-15T00:47:06,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/73ce752b758746dd9610e30de0fd3522, entries=150, sequenceid=529, filesize=12.0 K 2024-11-15T00:47:06,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/e02d983a3424402f8164667864ef703a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e02d983a3424402f8164667864ef703a 2024-11-15T00:47:06,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e02d983a3424402f8164667864ef703a, entries=150, sequenceid=529, filesize=12.0 K 2024-11-15T00:47:06,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/e7622390bac748edb2619206b6dc1dde as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/e7622390bac748edb2619206b6dc1dde 2024-11-15T00:47:06,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/e7622390bac748edb2619206b6dc1dde, entries=150, sequenceid=529, filesize=12.0 K 2024-11-15T00:47:06,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for 44c72b983f170fb79c2589a0cd8300ec in 863ms, sequenceid=529, compaction requested=false 2024-11-15T00:47:06,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 44c72b983f170fb79c2589a0cd8300ec: 2024-11-15T00:47:06,445 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:06,445 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:06,445 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. after waiting 0 ms 2024-11-15T00:47:06,445 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:06,445 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(2902): Flushing 44c72b983f170fb79c2589a0cd8300ec 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-15T00:47:06,445 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=A 2024-11-15T00:47:06,445 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:06,445 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=B 2024-11-15T00:47:06,445 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:06,445 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 44c72b983f170fb79c2589a0cd8300ec, store=C 2024-11-15T00:47:06,445 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:06,451 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/84712a6902694a749965c39a8b6f4ab0 is 50, key is test_row_0/A:col10/1731631625620/Put/seqid=0 2024-11-15T00:47:06,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742400_1576 (size=9857) 2024-11-15T00:47:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:47:06,855 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/84712a6902694a749965c39a8b6f4ab0 2024-11-15T00:47:06,862 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/5f520b4dfc4e4aee99096814a66cef6d is 50, key is test_row_0/B:col10/1731631625620/Put/seqid=0 2024-11-15T00:47:06,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742401_1577 (size=9857) 2024-11-15T00:47:07,272 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/5f520b4dfc4e4aee99096814a66cef6d 2024-11-15T00:47:07,282 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/bbe7bf0023fc4a34a87fc7fe19d2ab19 is 50, key is test_row_0/C:col10/1731631625620/Put/seqid=0 2024-11-15T00:47:07,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742402_1578 (size=9857) 2024-11-15T00:47:07,690 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/bbe7bf0023fc4a34a87fc7fe19d2ab19 2024-11-15T00:47:07,694 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/A/84712a6902694a749965c39a8b6f4ab0 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/84712a6902694a749965c39a8b6f4ab0 2024-11-15T00:47:07,697 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/84712a6902694a749965c39a8b6f4ab0, entries=100, sequenceid=536, filesize=9.6 K 2024-11-15T00:47:07,698 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/B/5f520b4dfc4e4aee99096814a66cef6d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5f520b4dfc4e4aee99096814a66cef6d 2024-11-15T00:47:07,701 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5f520b4dfc4e4aee99096814a66cef6d, entries=100, sequenceid=536, filesize=9.6 K 2024-11-15T00:47:07,701 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/.tmp/C/bbe7bf0023fc4a34a87fc7fe19d2ab19 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/bbe7bf0023fc4a34a87fc7fe19d2ab19 2024-11-15T00:47:07,704 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/bbe7bf0023fc4a34a87fc7fe19d2ab19, entries=100, sequenceid=536, filesize=9.6 K 2024-11-15T00:47:07,705 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 44c72b983f170fb79c2589a0cd8300ec in 1260ms, sequenceid=536, compaction requested=true 2024-11-15T00:47:07,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/3b2f83b9fe644105a22591e6c36784d9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/6d7e2729d0f54c1bb5d61bfd5cf9ac95, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/8be9b3122eb14b09ab634b7eb315c7dd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d3a0f7577fed49338171e36928f82f16, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fb6ebf5444da47fb87dcf46855feaa70, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/2e7470fad5eb42e4b5e672f63b53b544, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/69e2608a0daf415db2e672e12a0d7f5c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fc2ae3c97d034343ac17b574d96a925e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/bff037aa8b9b48dc8f5a02c9e9fb7e64, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/15052bc6520848bda864b28157f8b3a1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/17a239f299ce45e384ce9d9b5901b69f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/dbe7ded8f6114427b39ba5f3f34f6ca9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1bde81a675d7446bb720f391aff93a21, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/cc911e9e5a6e49cbbbdf5a2f37c088da, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/63a5e86f47e748179f389cef9ba7e403, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5a2ad7dc92524598a9a5a07f186a9d1e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ac82b27e70eb4ff69baac41068a1c414, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ec11f8f21b2f4d338104da38f815a4ff, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/4bf566d26f8940eb93af1b1a950f2a25, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/c1381988923e49ba8cfab965ca449534, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d2121678bbf54b3da5af177bbe00d1f2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d7325a73518f43d5becb54c7b6bd292a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b7aa6fc1d14b4c7fba23921a3cf5b0bf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1de0fdaca7454372811c7b831732563b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/584c830f99494b45acd3bf5dbddb8970, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5e6701aaa8ec429380da0660ec9a7ac9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ab555517644847df98e9da98ec7038e2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b6ab0fe9a97d48cd8f49c82114d49aef, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/811c8f5799f44444af258a4783c4e322, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/baaa5388cec44705be4cf6890eebb80d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/38f26e5cc96d4b238659a7b44cc04197, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5bc64c3a35524d44b2cdcf1fc7537ff0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/af121e39c3464d76b9ab0635ce7c431e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/0760c5166af744ed8781892953ddecee] to archive 2024-11-15T00:47:07,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:47:07,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/3b2f83b9fe644105a22591e6c36784d9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/3b2f83b9fe644105a22591e6c36784d9 2024-11-15T00:47:07,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/6d7e2729d0f54c1bb5d61bfd5cf9ac95 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/6d7e2729d0f54c1bb5d61bfd5cf9ac95 2024-11-15T00:47:07,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/8be9b3122eb14b09ab634b7eb315c7dd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/8be9b3122eb14b09ab634b7eb315c7dd 2024-11-15T00:47:07,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d3a0f7577fed49338171e36928f82f16 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d3a0f7577fed49338171e36928f82f16 2024-11-15T00:47:07,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fb6ebf5444da47fb87dcf46855feaa70 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fb6ebf5444da47fb87dcf46855feaa70 2024-11-15T00:47:07,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/2e7470fad5eb42e4b5e672f63b53b544 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/2e7470fad5eb42e4b5e672f63b53b544 2024-11-15T00:47:07,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/69e2608a0daf415db2e672e12a0d7f5c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/69e2608a0daf415db2e672e12a0d7f5c 2024-11-15T00:47:07,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fc2ae3c97d034343ac17b574d96a925e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/fc2ae3c97d034343ac17b574d96a925e 2024-11-15T00:47:07,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/bff037aa8b9b48dc8f5a02c9e9fb7e64 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/bff037aa8b9b48dc8f5a02c9e9fb7e64 2024-11-15T00:47:07,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/15052bc6520848bda864b28157f8b3a1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/15052bc6520848bda864b28157f8b3a1 2024-11-15T00:47:07,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/17a239f299ce45e384ce9d9b5901b69f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/17a239f299ce45e384ce9d9b5901b69f 2024-11-15T00:47:07,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/dbe7ded8f6114427b39ba5f3f34f6ca9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/dbe7ded8f6114427b39ba5f3f34f6ca9 2024-11-15T00:47:07,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1bde81a675d7446bb720f391aff93a21 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1bde81a675d7446bb720f391aff93a21 2024-11-15T00:47:07,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/cc911e9e5a6e49cbbbdf5a2f37c088da to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/cc911e9e5a6e49cbbbdf5a2f37c088da 2024-11-15T00:47:07,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/63a5e86f47e748179f389cef9ba7e403 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/63a5e86f47e748179f389cef9ba7e403 2024-11-15T00:47:07,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5a2ad7dc92524598a9a5a07f186a9d1e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5a2ad7dc92524598a9a5a07f186a9d1e 2024-11-15T00:47:07,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ac82b27e70eb4ff69baac41068a1c414 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ac82b27e70eb4ff69baac41068a1c414 2024-11-15T00:47:07,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ec11f8f21b2f4d338104da38f815a4ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ec11f8f21b2f4d338104da38f815a4ff 2024-11-15T00:47:07,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/4bf566d26f8940eb93af1b1a950f2a25 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/4bf566d26f8940eb93af1b1a950f2a25 2024-11-15T00:47:07,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/c1381988923e49ba8cfab965ca449534 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/c1381988923e49ba8cfab965ca449534 2024-11-15T00:47:07,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d2121678bbf54b3da5af177bbe00d1f2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d2121678bbf54b3da5af177bbe00d1f2 2024-11-15T00:47:07,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d7325a73518f43d5becb54c7b6bd292a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/d7325a73518f43d5becb54c7b6bd292a 2024-11-15T00:47:07,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b7aa6fc1d14b4c7fba23921a3cf5b0bf to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b7aa6fc1d14b4c7fba23921a3cf5b0bf 2024-11-15T00:47:07,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1de0fdaca7454372811c7b831732563b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/1de0fdaca7454372811c7b831732563b 2024-11-15T00:47:07,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/584c830f99494b45acd3bf5dbddb8970 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/584c830f99494b45acd3bf5dbddb8970 2024-11-15T00:47:07,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5e6701aaa8ec429380da0660ec9a7ac9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5e6701aaa8ec429380da0660ec9a7ac9 2024-11-15T00:47:07,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ab555517644847df98e9da98ec7038e2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/ab555517644847df98e9da98ec7038e2 2024-11-15T00:47:07,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b6ab0fe9a97d48cd8f49c82114d49aef to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/b6ab0fe9a97d48cd8f49c82114d49aef 2024-11-15T00:47:07,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/811c8f5799f44444af258a4783c4e322 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/811c8f5799f44444af258a4783c4e322 2024-11-15T00:47:07,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/baaa5388cec44705be4cf6890eebb80d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/baaa5388cec44705be4cf6890eebb80d 2024-11-15T00:47:07,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/38f26e5cc96d4b238659a7b44cc04197 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/38f26e5cc96d4b238659a7b44cc04197 2024-11-15T00:47:07,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5bc64c3a35524d44b2cdcf1fc7537ff0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/5bc64c3a35524d44b2cdcf1fc7537ff0 2024-11-15T00:47:07,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/af121e39c3464d76b9ab0635ce7c431e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/af121e39c3464d76b9ab0635ce7c431e 2024-11-15T00:47:07,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/0760c5166af744ed8781892953ddecee to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/0760c5166af744ed8781892953ddecee 2024-11-15T00:47:07,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fd6273e76a0f417bb6a024a9fb842fed, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/7e08e0fc17d3413fa327d80ed37135ae, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/395b2443356241d4992e0a7a57c81131, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f697eca52bd340568528fc39546a71bc, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/797e9d10b33d4aeaadfc5dc0e85a0bef, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c8f79f978812405db0208ae6662c77a9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/42b2f97d3b224b5ba888813f854df6cd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/099c27874fc24f7b8284449dc16d43e6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/9742f897a26f4c02b575f192e2a15a05, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/691582fe09ef47ca8b49dc3aa2df5b88, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/714ff9546f3b45ddae2f42b6c6201435, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e6c45569440d4a2ea2120bf451e76334, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3b1a75be73df46f39ac26c5f1bb8932f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e0247a9e6c1f404297f10938db06b4f4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fe69281047c94425b3a62947b6965003, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4ec9c3e83b83428aa89754d462980948, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/db8234b0abf04b7fa78e8187cf81e5cd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/47bec14a7f074f1088074e360592fcf1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/867c9f9436b448a5938790abcd990583, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5b84e5f94a594dccbb47f8cfcf4bf29a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/d0108e48aadd427db711e007588e42e4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4f361492da1b4f5fa44e8a4307656bee, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/596ab698b3844962a4c8eb355c04aad7, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c2ed8c4e92a647eda2bbbe2ff7b179b0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/8c91b7f54c814b00af38b082c561be31, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f745a6ee2c7447aaa71fdeed29afc547, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5dc79f8f9fc54b66b1baf86a46efe7cd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c44ae27570034a899c4e93b176fca72e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3bc67725f6514890bcf865ddb6f339d2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/44041b5af20648aa9ecabf3bdd65c5ae, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/04640945aa8246cbaf31ee9159467fe4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/819fd8254c594f289f9d377f41b7f672, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/ffb9710a3cc44491b58e5ee7dfeb7055, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/b3c30af6f9a1481cb157b722c158e9fb] to archive 2024-11-15T00:47:07,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:47:07,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fd6273e76a0f417bb6a024a9fb842fed to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fd6273e76a0f417bb6a024a9fb842fed 2024-11-15T00:47:07,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/7e08e0fc17d3413fa327d80ed37135ae to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/7e08e0fc17d3413fa327d80ed37135ae 2024-11-15T00:47:07,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/395b2443356241d4992e0a7a57c81131 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/395b2443356241d4992e0a7a57c81131 2024-11-15T00:47:07,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f697eca52bd340568528fc39546a71bc to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f697eca52bd340568528fc39546a71bc 2024-11-15T00:47:07,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/797e9d10b33d4aeaadfc5dc0e85a0bef to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/797e9d10b33d4aeaadfc5dc0e85a0bef 2024-11-15T00:47:07,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c8f79f978812405db0208ae6662c77a9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c8f79f978812405db0208ae6662c77a9 2024-11-15T00:47:07,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/42b2f97d3b224b5ba888813f854df6cd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/42b2f97d3b224b5ba888813f854df6cd 2024-11-15T00:47:07,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/099c27874fc24f7b8284449dc16d43e6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/099c27874fc24f7b8284449dc16d43e6 2024-11-15T00:47:07,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/9742f897a26f4c02b575f192e2a15a05 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/9742f897a26f4c02b575f192e2a15a05 2024-11-15T00:47:07,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/691582fe09ef47ca8b49dc3aa2df5b88 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/691582fe09ef47ca8b49dc3aa2df5b88 2024-11-15T00:47:07,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/714ff9546f3b45ddae2f42b6c6201435 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/714ff9546f3b45ddae2f42b6c6201435 2024-11-15T00:47:07,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e6c45569440d4a2ea2120bf451e76334 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e6c45569440d4a2ea2120bf451e76334 2024-11-15T00:47:07,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3b1a75be73df46f39ac26c5f1bb8932f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3b1a75be73df46f39ac26c5f1bb8932f 2024-11-15T00:47:07,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e0247a9e6c1f404297f10938db06b4f4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e0247a9e6c1f404297f10938db06b4f4 2024-11-15T00:47:07,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fe69281047c94425b3a62947b6965003 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/fe69281047c94425b3a62947b6965003 2024-11-15T00:47:07,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4ec9c3e83b83428aa89754d462980948 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4ec9c3e83b83428aa89754d462980948 2024-11-15T00:47:07,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/db8234b0abf04b7fa78e8187cf81e5cd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/db8234b0abf04b7fa78e8187cf81e5cd 2024-11-15T00:47:07,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/47bec14a7f074f1088074e360592fcf1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/47bec14a7f074f1088074e360592fcf1 2024-11-15T00:47:07,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/867c9f9436b448a5938790abcd990583 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/867c9f9436b448a5938790abcd990583 2024-11-15T00:47:07,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5b84e5f94a594dccbb47f8cfcf4bf29a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5b84e5f94a594dccbb47f8cfcf4bf29a 2024-11-15T00:47:07,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/d0108e48aadd427db711e007588e42e4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/d0108e48aadd427db711e007588e42e4 2024-11-15T00:47:07,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4f361492da1b4f5fa44e8a4307656bee to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/4f361492da1b4f5fa44e8a4307656bee 2024-11-15T00:47:07,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/596ab698b3844962a4c8eb355c04aad7 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/596ab698b3844962a4c8eb355c04aad7 2024-11-15T00:47:07,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c2ed8c4e92a647eda2bbbe2ff7b179b0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c2ed8c4e92a647eda2bbbe2ff7b179b0 2024-11-15T00:47:07,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/8c91b7f54c814b00af38b082c561be31 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/8c91b7f54c814b00af38b082c561be31 2024-11-15T00:47:07,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f745a6ee2c7447aaa71fdeed29afc547 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/f745a6ee2c7447aaa71fdeed29afc547 2024-11-15T00:47:07,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5dc79f8f9fc54b66b1baf86a46efe7cd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5dc79f8f9fc54b66b1baf86a46efe7cd 2024-11-15T00:47:07,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c44ae27570034a899c4e93b176fca72e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/c44ae27570034a899c4e93b176fca72e 2024-11-15T00:47:07,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3bc67725f6514890bcf865ddb6f339d2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3bc67725f6514890bcf865ddb6f339d2 2024-11-15T00:47:07,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/44041b5af20648aa9ecabf3bdd65c5ae to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/44041b5af20648aa9ecabf3bdd65c5ae 2024-11-15T00:47:07,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/04640945aa8246cbaf31ee9159467fe4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/04640945aa8246cbaf31ee9159467fe4 2024-11-15T00:47:07,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/819fd8254c594f289f9d377f41b7f672 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/819fd8254c594f289f9d377f41b7f672 2024-11-15T00:47:07,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/ffb9710a3cc44491b58e5ee7dfeb7055 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/ffb9710a3cc44491b58e5ee7dfeb7055 2024-11-15T00:47:07,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/b3c30af6f9a1481cb157b722c158e9fb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/b3c30af6f9a1481cb157b722c158e9fb 2024-11-15T00:47:07,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/53a2e1d1528a481e82ecb2a1d0d5ce6e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/8c37464f1ed24f4e9a9d1c21adf1ace9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/563eb5b041c14db6bb4edcebb4539c06, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3e022fbb5da147c099611f2eed329cac, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/bcd11b4d556346218eacc4f4478fd812, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f24bd1b4a879434ebf64589b1eb46bfe, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b04f5d743f154096ab7075718e4fcd78, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/02139429135949a8bf2f0488023a75ff, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6746b82dc27f4a40b0647caffdc6ded7, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f9c926ab835245eeb816874722ebce58, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3486a91134b1456aaa3e867606b03287, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/594d4ebbe7b94163b0721b13e73d6729, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/10f8e84075ce42248b2b1b09968fb838, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/5e72f66d407c4d6b97ac44864b0f4410, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/d88f2a1bd51142c984739a4563e06f57, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/068fa635e81c43f8bdb79c03bd938a82, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/be6989162633492a9dbf11839d1d738a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/4727ec47c0634391bb397270c226df05, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/72df85411ca144a8a80b0c562b1af6a2, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9a76ef34c14c43ab8df55029f97f0958, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/acac18f00f16457f92bc90ef3d7e58a0, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/ce65316e7475481db0b082e7d18b8a19, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b66b7dd44d5841ce891a1bc5b1d5fccf, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/04a83eb50a5c4561b450b26a655ea15e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/abf97a304ab64e10a449c0d43714ef1a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6a50d8475f124e8b999efe5d813f3370, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cdf5e3df9b2e4d3d8d4934646c055f10, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/986b112432c746d99f1655c60a067c29, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b70a4a60b9c4445eb624a0b9dfa4ddba, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9d959466c4cb49e4818358c206ff5d4b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cb59c7a967614859a0cb7fe0d1c494a3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b29071fd80714be1b82be5daa963e851, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/24ef4c4f503445e0bae1c36e745a7e96, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/2546bc770e294b85a39a0cd01762410b] to archive 2024-11-15T00:47:07,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:47:07,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/53a2e1d1528a481e82ecb2a1d0d5ce6e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/53a2e1d1528a481e82ecb2a1d0d5ce6e 2024-11-15T00:47:07,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/8c37464f1ed24f4e9a9d1c21adf1ace9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/8c37464f1ed24f4e9a9d1c21adf1ace9 2024-11-15T00:47:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:47:07,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/563eb5b041c14db6bb4edcebb4539c06 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/563eb5b041c14db6bb4edcebb4539c06 2024-11-15T00:47:07,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3e022fbb5da147c099611f2eed329cac to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3e022fbb5da147c099611f2eed329cac 2024-11-15T00:47:07,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/bcd11b4d556346218eacc4f4478fd812 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/bcd11b4d556346218eacc4f4478fd812 2024-11-15T00:47:07,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f24bd1b4a879434ebf64589b1eb46bfe to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f24bd1b4a879434ebf64589b1eb46bfe 2024-11-15T00:47:07,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b04f5d743f154096ab7075718e4fcd78 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b04f5d743f154096ab7075718e4fcd78 2024-11-15T00:47:07,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/02139429135949a8bf2f0488023a75ff to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/02139429135949a8bf2f0488023a75ff 2024-11-15T00:47:07,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6746b82dc27f4a40b0647caffdc6ded7 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6746b82dc27f4a40b0647caffdc6ded7 2024-11-15T00:47:07,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f9c926ab835245eeb816874722ebce58 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/f9c926ab835245eeb816874722ebce58 2024-11-15T00:47:07,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3486a91134b1456aaa3e867606b03287 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/3486a91134b1456aaa3e867606b03287 2024-11-15T00:47:07,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/594d4ebbe7b94163b0721b13e73d6729 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/594d4ebbe7b94163b0721b13e73d6729 2024-11-15T00:47:07,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/10f8e84075ce42248b2b1b09968fb838 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/10f8e84075ce42248b2b1b09968fb838 2024-11-15T00:47:07,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/5e72f66d407c4d6b97ac44864b0f4410 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/5e72f66d407c4d6b97ac44864b0f4410 2024-11-15T00:47:07,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/d88f2a1bd51142c984739a4563e06f57 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/d88f2a1bd51142c984739a4563e06f57 2024-11-15T00:47:07,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/068fa635e81c43f8bdb79c03bd938a82 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/068fa635e81c43f8bdb79c03bd938a82 2024-11-15T00:47:07,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/be6989162633492a9dbf11839d1d738a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/be6989162633492a9dbf11839d1d738a 2024-11-15T00:47:07,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/4727ec47c0634391bb397270c226df05 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/4727ec47c0634391bb397270c226df05 2024-11-15T00:47:07,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/72df85411ca144a8a80b0c562b1af6a2 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/72df85411ca144a8a80b0c562b1af6a2 2024-11-15T00:47:07,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9a76ef34c14c43ab8df55029f97f0958 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9a76ef34c14c43ab8df55029f97f0958 2024-11-15T00:47:07,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/acac18f00f16457f92bc90ef3d7e58a0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/acac18f00f16457f92bc90ef3d7e58a0 2024-11-15T00:47:07,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/ce65316e7475481db0b082e7d18b8a19 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/ce65316e7475481db0b082e7d18b8a19 2024-11-15T00:47:07,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b66b7dd44d5841ce891a1bc5b1d5fccf to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b66b7dd44d5841ce891a1bc5b1d5fccf 2024-11-15T00:47:07,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/04a83eb50a5c4561b450b26a655ea15e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/04a83eb50a5c4561b450b26a655ea15e 2024-11-15T00:47:07,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/abf97a304ab64e10a449c0d43714ef1a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/abf97a304ab64e10a449c0d43714ef1a 2024-11-15T00:47:07,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6a50d8475f124e8b999efe5d813f3370 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/6a50d8475f124e8b999efe5d813f3370 2024-11-15T00:47:07,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cdf5e3df9b2e4d3d8d4934646c055f10 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cdf5e3df9b2e4d3d8d4934646c055f10 2024-11-15T00:47:07,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/986b112432c746d99f1655c60a067c29 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/986b112432c746d99f1655c60a067c29 2024-11-15T00:47:07,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b70a4a60b9c4445eb624a0b9dfa4ddba to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b70a4a60b9c4445eb624a0b9dfa4ddba 2024-11-15T00:47:07,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9d959466c4cb49e4818358c206ff5d4b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/9d959466c4cb49e4818358c206ff5d4b 2024-11-15T00:47:07,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cb59c7a967614859a0cb7fe0d1c494a3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/cb59c7a967614859a0cb7fe0d1c494a3 2024-11-15T00:47:07,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b29071fd80714be1b82be5daa963e851 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/b29071fd80714be1b82be5daa963e851 2024-11-15T00:47:07,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/24ef4c4f503445e0bae1c36e745a7e96 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/24ef4c4f503445e0bae1c36e745a7e96 2024-11-15T00:47:07,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/2546bc770e294b85a39a0cd01762410b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/2546bc770e294b85a39a0cd01762410b 2024-11-15T00:47:07,824 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/recovered.edits/539.seqid, newMaxSeqId=539, maxSeqId=1 2024-11-15T00:47:07,824 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. 2024-11-15T00:47:07,824 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1676): Region close journal for 44c72b983f170fb79c2589a0cd8300ec: Waiting for close lock at 1731631625788Running coprocessor pre-close hooks at 1731631625788Disabling compacts and flushes for region at 1731631625788Disabling writes for close at 1731631626445 (+657 ms)Obtaining lock to block concurrent updates at 1731631626445Preparing flush snapshotting stores in 44c72b983f170fb79c2589a0cd8300ec at 1731631626445Finished memstore snapshotting TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731631626445Flushing stores of TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec. at 1731631626448 (+3 ms)Flushing 44c72b983f170fb79c2589a0cd8300ec/A: creating writer at 1731631626449 (+1 ms)Flushing 44c72b983f170fb79c2589a0cd8300ec/A: appending metadata at 1731631626451 (+2 ms)Flushing 44c72b983f170fb79c2589a0cd8300ec/A: closing flushed file at 1731631626451Flushing 44c72b983f170fb79c2589a0cd8300ec/B: creating writer at 1731631626860 (+409 ms)Flushing 44c72b983f170fb79c2589a0cd8300ec/B: appending metadata at 1731631626862 (+2 ms)Flushing 44c72b983f170fb79c2589a0cd8300ec/B: closing flushed file at 1731631626862Flushing 44c72b983f170fb79c2589a0cd8300ec/C: creating writer at 1731631627276 (+414 ms)Flushing 44c72b983f170fb79c2589a0cd8300ec/C: appending metadata at 1731631627281 (+5 ms)Flushing 44c72b983f170fb79c2589a0cd8300ec/C: closing flushed file at 1731631627281Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7514fc2d: reopening flushed file at 1731631627693 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c3d8418: reopening flushed file at 1731631627697 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@420f9f30: reopening flushed file at 1731631627701 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 44c72b983f170fb79c2589a0cd8300ec in 1260ms, sequenceid=536, compaction requested=true at 1731631627705 (+4 ms)Writing region close event to WAL at 1731631627821 (+116 ms)Running coprocessor post-close hooks at 1731631627824 (+3 ms)Closed at 1731631627824 2024-11-15T00:47:07,826 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(157): Closed 44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:07,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=44c72b983f170fb79c2589a0cd8300ec, regionState=CLOSED 2024-11-15T00:47:07,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:47:07,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-15T00:47:07,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseRegionProcedure 44c72b983f170fb79c2589a0cd8300ec, server=fb2f6cb7846a,34537,1731631472208 in 2.1940 sec 2024-11-15T00:47:07,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-15T00:47:07,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=44c72b983f170fb79c2589a0cd8300ec, UNASSIGN in 2.1990 sec 2024-11-15T00:47:07,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-15T00:47:07,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2030 sec 2024-11-15T00:47:07,834 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631627834"}]},"ts":"1731631627834"} 2024-11-15T00:47:07,836 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:47:07,836 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:47:07,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2120 sec 2024-11-15T00:47:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:47:09,800 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:09,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:47:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:09,801 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T00:47:09,802 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=129, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:09,804 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:09,805 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/recovered.edits] 2024-11-15T00:47:09,807 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/08e9941d8abf47318ff99d8b1a84aba3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/08e9941d8abf47318ff99d8b1a84aba3 2024-11-15T00:47:09,809 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/73ce752b758746dd9610e30de0fd3522 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/73ce752b758746dd9610e30de0fd3522 2024-11-15T00:47:09,809 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/84712a6902694a749965c39a8b6f4ab0 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/A/84712a6902694a749965c39a8b6f4ab0 2024-11-15T00:47:09,811 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3f49bbbc5cae4a78a6cd77ce1296db80 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/3f49bbbc5cae4a78a6cd77ce1296db80 2024-11-15T00:47:09,812 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5f520b4dfc4e4aee99096814a66cef6d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/5f520b4dfc4e4aee99096814a66cef6d 2024-11-15T00:47:09,813 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e02d983a3424402f8164667864ef703a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/B/e02d983a3424402f8164667864ef703a 2024-11-15T00:47:09,815 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/bbe7bf0023fc4a34a87fc7fe19d2ab19 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/bbe7bf0023fc4a34a87fc7fe19d2ab19 2024-11-15T00:47:09,816 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/c5fa43fbed714f779341e080db1be9ba to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/c5fa43fbed714f779341e080db1be9ba 2024-11-15T00:47:09,817 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/e7622390bac748edb2619206b6dc1dde to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/C/e7622390bac748edb2619206b6dc1dde 2024-11-15T00:47:09,820 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/recovered.edits/539.seqid to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec/recovered.edits/539.seqid 2024-11-15T00:47:09,821 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/44c72b983f170fb79c2589a0cd8300ec 2024-11-15T00:47:09,821 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:47:09,822 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=129, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:09,824 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:47:09,827 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:47:09,828 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=129, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:09,828 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:47:09,828 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631629828"}]},"ts":"9223372036854775807"} 2024-11-15T00:47:09,831 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:47:09,831 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 44c72b983f170fb79c2589a0cd8300ec, NAME => 'TestAcidGuarantees,,1731631601296.44c72b983f170fb79c2589a0cd8300ec.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:47:09,831 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:47:09,831 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631629831"}]},"ts":"9223372036854775807"} 2024-11-15T00:47:09,833 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:47:09,836 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=129, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:09,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-11-15T00:47:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T00:47:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:47:09,913 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:09,919 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 250), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1070 (was 1166), ProcessCount=11 (was 11), AvailableMemoryMB=1995 (was 2202) 2024-11-15T00:47:09,928 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1070, ProcessCount=11, AvailableMemoryMB=1993 2024-11-15T00:47:09,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:47:09,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:47:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:09,931 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:47:09,931 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:09,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 130 2024-11-15T00:47:09,932 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:47:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-15T00:47:09,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742403_1579 (size=1061) 2024-11-15T00:47:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-15T00:47:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-15T00:47:10,338 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403 2024-11-15T00:47:10,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742404_1580 (size=53) 2024-11-15T00:47:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-15T00:47:10,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:47:10,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8072ff2efb6653ab0fbbcdf205cf0e99, disabling compactions & flushes 2024-11-15T00:47:10,753 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:10,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:10,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. after waiting 0 ms 2024-11-15T00:47:10,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:10,753 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:10,753 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8072ff2efb6653ab0fbbcdf205cf0e99: Waiting for close lock at 1731631630753Disabling compacts and flushes for region at 1731631630753Disabling writes for close at 1731631630753Writing region close event to WAL at 1731631630753Closed at 1731631630753 2024-11-15T00:47:10,755 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:47:10,755 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631630755"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631630755"}]},"ts":"1731631630755"} 2024-11-15T00:47:10,757 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:47:10,758 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:47:10,758 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631630758"}]},"ts":"1731631630758"} 2024-11-15T00:47:10,760 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:47:10,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, ASSIGN}] 2024-11-15T00:47:10,762 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, ASSIGN 2024-11-15T00:47:10,763 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,34537,1731631472208; forceNewPlan=false, retain=false 2024-11-15T00:47:10,915 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=8072ff2efb6653ab0fbbcdf205cf0e99, regionState=OPENING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:10,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, ASSIGN because future has completed 2024-11-15T00:47:10,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:47:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-15T00:47:11,071 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:11,071 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7752): Opening region: {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:47:11,072 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,072 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:47:11,072 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7794): checking encryption for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,072 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7797): checking classloading for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,073 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,074 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:47:11,074 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8072ff2efb6653ab0fbbcdf205cf0e99 columnFamilyName A 2024-11-15T00:47:11,074 DEBUG [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:11,074 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(327): Store=8072ff2efb6653ab0fbbcdf205cf0e99/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:47:11,075 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,076 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:47:11,076 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8072ff2efb6653ab0fbbcdf205cf0e99 columnFamilyName B 2024-11-15T00:47:11,076 DEBUG [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:11,084 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(327): Store=8072ff2efb6653ab0fbbcdf205cf0e99/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:47:11,084 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,085 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:47:11,086 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8072ff2efb6653ab0fbbcdf205cf0e99 columnFamilyName C 2024-11-15T00:47:11,086 DEBUG [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:11,086 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(327): Store=8072ff2efb6653ab0fbbcdf205cf0e99/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:47:11,086 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:11,086 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1038): replaying wal for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,087 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,087 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,088 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1048): stopping wal replay for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,088 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1060): Cleaning up temporary data for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,088 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:47:11,089 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): writing seq id for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,091 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:47:11,091 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1114): Opened 8072ff2efb6653ab0fbbcdf205cf0e99; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66309237, jitterRate=-0.01191537082195282}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:47:11,091 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:11,092 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1006): Region open journal for 8072ff2efb6653ab0fbbcdf205cf0e99: Running coprocessor pre-open hook at 1731631631072Writing region info on filesystem at 1731631631072Initializing all the Stores at 1731631631072Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631631073 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631631073Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631631073Cleaning up temporary data from old regions at 1731631631088 (+15 ms)Running coprocessor post-open hooks at 1731631631091 (+3 ms)Region opened successfully at 1731631631091 2024-11-15T00:47:11,092 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., pid=132, masterSystemTime=1731631631068 2024-11-15T00:47:11,093 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:11,094 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:11,094 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=8072ff2efb6653ab0fbbcdf205cf0e99, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:11,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:47:11,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-15T00:47:11,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; OpenRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 in 178 msec 2024-11-15T00:47:11,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-15T00:47:11,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, ASSIGN in 336 msec 2024-11-15T00:47:11,099 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:47:11,099 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631631099"}]},"ts":"1731631631099"} 2024-11-15T00:47:11,100 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:47:11,101 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:47:11,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-15T00:47:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-15T00:47:12,090 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:12,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:12,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:12,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:12,093 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71bb66f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:12,095 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ec36045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:12,096 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:12,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:12,099 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:47:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,33521,1731631471305 2024-11-15T00:47:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47fbb7e5 2024-11-15T00:47:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:47:12,101 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:47:12,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:47:12,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:47:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:12,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742405_1581 (size=1097) 2024-11-15T00:47:12,514 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T00:47:12,514 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T00:47:12,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:47:12,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, REOPEN/MOVE}] 2024-11-15T00:47:12,520 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, REOPEN/MOVE 2024-11-15T00:47:12,521 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=8072ff2efb6653ab0fbbcdf205cf0e99, regionState=CLOSING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:12,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, REOPEN/MOVE because future has completed 2024-11-15T00:47:12,523 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:47:12,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:47:12,676 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:12,676 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:47:12,676 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 8072ff2efb6653ab0fbbcdf205cf0e99, disabling compactions & flushes 2024-11-15T00:47:12,676 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:12,676 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:12,676 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. after waiting 0 ms 2024-11-15T00:47:12,676 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:12,681 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:47:12,681 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:12,681 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 8072ff2efb6653ab0fbbcdf205cf0e99: Waiting for close lock at 1731631632676Running coprocessor pre-close hooks at 1731631632676Disabling compacts and flushes for region at 1731631632676Disabling writes for close at 1731631632676Writing region close event to WAL at 1731631632676Running coprocessor post-close hooks at 1731631632681 (+5 ms)Closed at 1731631632681 2024-11-15T00:47:12,682 WARN [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionServer(3299): Not adding moved region record: 8072ff2efb6653ab0fbbcdf205cf0e99 to self. 2024-11-15T00:47:12,684 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:12,685 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=8072ff2efb6653ab0fbbcdf205cf0e99, regionState=CLOSED 2024-11-15T00:47:12,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:47:12,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-15T00:47:12,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 in 165 msec 2024-11-15T00:47:12,691 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, REOPEN/MOVE; state=CLOSED, location=fb2f6cb7846a,34537,1731631472208; forceNewPlan=false, retain=true 2024-11-15T00:47:12,842 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=8072ff2efb6653ab0fbbcdf205cf0e99, regionState=OPENING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:12,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, REOPEN/MOVE because future has completed 2024-11-15T00:47:12,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:47:12,997 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:12,997 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7752): Opening region: {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:47:12,998 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:12,998 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:47:12,998 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7794): checking encryption for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:12,998 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7797): checking classloading for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:12,999 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,000 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:47:13,001 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8072ff2efb6653ab0fbbcdf205cf0e99 columnFamilyName A 2024-11-15T00:47:13,002 DEBUG [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:13,003 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(327): Store=8072ff2efb6653ab0fbbcdf205cf0e99/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:47:13,004 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,004 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:47:13,004 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8072ff2efb6653ab0fbbcdf205cf0e99 columnFamilyName B 2024-11-15T00:47:13,004 DEBUG [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:13,005 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(327): Store=8072ff2efb6653ab0fbbcdf205cf0e99/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:47:13,005 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,005 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T00:47:13,005 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8072ff2efb6653ab0fbbcdf205cf0e99 columnFamilyName C 2024-11-15T00:47:13,005 DEBUG [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:13,005 INFO [StoreOpener-8072ff2efb6653ab0fbbcdf205cf0e99-1 {}] regionserver.HStore(327): Store=8072ff2efb6653ab0fbbcdf205cf0e99/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:47:13,006 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,006 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1038): replaying wal for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,006 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,007 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,008 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1048): stopping wal replay for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,008 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1060): Cleaning up temporary data for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,008 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:47:13,009 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1093): writing seq id for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,009 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1114): Opened 8072ff2efb6653ab0fbbcdf205cf0e99; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66628090, jitterRate=-0.007164090871810913}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:47:13,009 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,010 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1006): Region open journal for 8072ff2efb6653ab0fbbcdf205cf0e99: Running coprocessor pre-open hook at 1731631632998Writing region info on filesystem at 1731631632998Initializing all the Stores at 1731631632999 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631632999Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631632999Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631632999Cleaning up temporary data from old regions at 1731631633008 (+9 ms)Running coprocessor post-open hooks at 1731631633009 (+1 ms)Region opened successfully at 1731631633010 (+1 ms) 2024-11-15T00:47:13,010 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., pid=137, masterSystemTime=1731631632994 2024-11-15T00:47:13,011 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,012 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,012 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=8072ff2efb6653ab0fbbcdf205cf0e99, regionState=OPEN, openSeqNum=5, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:47:13,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=135 2024-11-15T00:47:13,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 in 170 msec 2024-11-15T00:47:13,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-15T00:47:13,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, REOPEN/MOVE in 496 msec 2024-11-15T00:47:13,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-15T00:47:13,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-15T00:47:13,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 915 msec 2024-11-15T00:47:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T00:47:13,021 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:13,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:13,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:13,022 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205611a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:13,023 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2144d4b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:13,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:13,025 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2399f8ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:13,026 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16611098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:13,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:13,028 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb1e996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:13,029 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b124bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:13,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:13,030 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6236e72a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:13,031 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dcaabed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:13,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:13,033 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dcc559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:13,034 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d72da1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:13,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:13,035 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f763b67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:13,036 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6973ca39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:13,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:13,037 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a379ce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:13,039 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fee1b8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:13,040 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:13,041 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23605aa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:13,042 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@faa4a09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:13,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:13,044 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ef1f06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:13,045 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a734ae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,33521,-1 for getting cluster id 2024-11-15T00:47:13,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:47:13,047 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '78911300-2cd1-4362-aa5b-886d0f47db71' 2024-11-15T00:47:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:47:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "78911300-2cd1-4362-aa5b-886d0f47db71" 2024-11-15T00:47:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@576abb59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,33521,-1] 2024-11-15T00:47:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:47:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:13,048 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:47:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15051d70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:47:13,049 DEBUG [Thread-2592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:13,049 DEBUG [Thread-2596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:13,049 DEBUG [Thread-2590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:13,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:47:13,049 DEBUG [Thread-2581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:13,050 DEBUG [Thread-2585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:13,050 DEBUG [Thread-2588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-11-15T00:47:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:13,051 DEBUG [Thread-2594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:13,051 DEBUG [Thread-2577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:13,052 DEBUG [Thread-2583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:13,052 DEBUG [Thread-2579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:47:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:13,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:13,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:13,053 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:47:13,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:13,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:13,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T00:47:13,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:47:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:47:13,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,34537,1731631472208, seqNum=-1] 2024-11-15T00:47:13,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:47:13,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:47:13,057 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:47:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:47:13,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:47:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:47:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:47:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:47:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:47:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:47:13,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:47:13,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5] 2024-11-15T00:47:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T00:47:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:13,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115516c5b73d73449478d271748e6fdab34_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631633063/Put/seqid=0 2024-11-15T00:47:13,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742406_1582 (size=14594) 2024-11-15T00:47:13,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631693124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631693124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631693125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631633085, max timestamp=1731631633129Num uniques 859; 2024-11-15T00:47:13,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631633085, max timestamp=1731631633129Num uniques 859; 2024-11-15T00:47:13,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631633085, max timestamp=1731631633129Num uniques 859; 2024-11-15T00:47:13,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:13,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:13,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:13,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T00:47:13,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631693162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631693165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-15T00:47:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631693239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631693239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631693239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631693279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631693279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,347 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:47:13,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-15T00:47:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T00:47:13,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631693449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631693449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631693449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631693489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631693490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,496 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:13,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115516c5b73d73449478d271748e6fdab34_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115516c5b73d73449478d271748e6fdab34_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:13,500 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/22638295d9d14264941a09fb892a63cb, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:13,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/22638295d9d14264941a09fb892a63cb is 175, key is test_row_0/A:col10/1731631633063/Put/seqid=0 2024-11-15T00:47:13,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742407_1583 (size=39549) 2024-11-15T00:47:13,504 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/22638295d9d14264941a09fb892a63cb 2024-11-15T00:47:13,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-15T00:47:13,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:13,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/3a499b6902894fffbe00dad0998fe683 is 50, key is test_row_0/B:col10/1731631633063/Put/seqid=0 2024-11-15T00:47:13,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742408_1584 (size=12001) 2024-11-15T00:47:13,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-15T00:47:13,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:13,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T00:47:13,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631693759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631693759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631693760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631693799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:13,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631693799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:13,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:13,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:13,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-15T00:47:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-15T00:47:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:13,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:14,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-15T00:47:14,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:14,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:14,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:14,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T00:47:14,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/3a499b6902894fffbe00dad0998fe683 2024-11-15T00:47:14,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:14,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631694269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631694269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:14,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631694270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:14,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/01e40590fd854a6da7ae9d410ce5a2c6 is 50, key is test_row_0/C:col10/1731631633063/Put/seqid=0 2024-11-15T00:47:14,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742409_1585 (size=12001) 2024-11-15T00:47:14,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/01e40590fd854a6da7ae9d410ce5a2c6 2024-11-15T00:47:14,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/22638295d9d14264941a09fb892a63cb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/22638295d9d14264941a09fb892a63cb 2024-11-15T00:47:14,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-15T00:47:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:14,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:14,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/22638295d9d14264941a09fb892a63cb, entries=200, sequenceid=15, filesize=38.6 K 2024-11-15T00:47:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:14,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/3a499b6902894fffbe00dad0998fe683 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/3a499b6902894fffbe00dad0998fe683 2024-11-15T00:47:14,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631694309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:14,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/3a499b6902894fffbe00dad0998fe683, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T00:47:14,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/01e40590fd854a6da7ae9d410ce5a2c6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/01e40590fd854a6da7ae9d410ce5a2c6 2024-11-15T00:47:14,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631694309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:14,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/01e40590fd854a6da7ae9d410ce5a2c6, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T00:47:14,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1253ms, sequenceid=15, compaction requested=false 2024-11-15T00:47:14,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:14,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-15T00:47:14,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:14,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-15T00:47:14,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:14,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:14,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:14,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:14,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:14,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:14,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150b90835d06084799b09f24ba11e7755a_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631633163/Put/seqid=0 2024-11-15T00:47:14,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742410_1586 (size=12154) 2024-11-15T00:47:14,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:14,480 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150b90835d06084799b09f24ba11e7755a_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150b90835d06084799b09f24ba11e7755a_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:14,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/0d7cdac77b914e529903734c8e15316b, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:14,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/0d7cdac77b914e529903734c8e15316b is 175, key is test_row_0/A:col10/1731631633163/Put/seqid=0 2024-11-15T00:47:14,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742411_1587 (size=30955) 2024-11-15T00:47:14,904 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/0d7cdac77b914e529903734c8e15316b 2024-11-15T00:47:14,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/c8799593c5e4436e95140eb627c67e72 is 50, key is test_row_0/B:col10/1731631633163/Put/seqid=0 2024-11-15T00:47:14,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742412_1588 (size=12001) 2024-11-15T00:47:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T00:47:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:15,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:15,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631695295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631695298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631695299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631695319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,321 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in 8072ff2efb6653ab0fbbcdf205cf0e99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:47:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631695329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,332 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in 8072ff2efb6653ab0fbbcdf205cf0e99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:47:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,344 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/c8799593c5e4436e95140eb627c67e72 2024-11-15T00:47:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/fdca5f98b3ee40aa90590d4998803df4 is 50, key is test_row_0/C:col10/1731631633163/Put/seqid=0 2024-11-15T00:47:15,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742413_1589 (size=12001) 2024-11-15T00:47:15,376 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/fdca5f98b3ee40aa90590d4998803df4 2024-11-15T00:47:15,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/0d7cdac77b914e529903734c8e15316b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0d7cdac77b914e529903734c8e15316b 2024-11-15T00:47:15,396 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0d7cdac77b914e529903734c8e15316b, entries=150, sequenceid=51, filesize=30.2 K 2024-11-15T00:47:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/c8799593c5e4436e95140eb627c67e72 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/c8799593c5e4436e95140eb627c67e72 2024-11-15T00:47:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,407 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/c8799593c5e4436e95140eb627c67e72, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T00:47:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/fdca5f98b3ee40aa90590d4998803df4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/fdca5f98b3ee40aa90590d4998803df4 2024-11-15T00:47:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631695409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631695410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631695410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,417 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/fdca5f98b3ee40aa90590d4998803df4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T00:47:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,418 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 963ms, sequenceid=51, compaction requested=false 2024-11-15T00:47:15,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:15,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:15,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-11-15T00:47:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-11-15T00:47:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-15T00:47:15,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3670 sec 2024-11-15T00:47:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 2.3820 sec 2024-11-15T00:47:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:15,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:47:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:15,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:15,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:15,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:15,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:15,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157545d0091cbd45f0980e89ec5dd61153_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631635624/Put/seqid=0 2024-11-15T00:47:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742414_1590 (size=14594) 2024-11-15T00:47:15,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,672 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157545d0091cbd45f0980e89ec5dd61153_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157545d0091cbd45f0980e89ec5dd61153_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:15,674 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/2e082444302645c797fe05148e744b29, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:15,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/2e082444302645c797fe05148e744b29 is 175, key is test_row_0/A:col10/1731631635624/Put/seqid=0 2024-11-15T00:47:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631695682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631695685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631695686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742415_1591 (size=39545) 2024-11-15T00:47:15,732 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/2e082444302645c797fe05148e744b29 2024-11-15T00:47:15,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/f2f96ac7ae344160ab9267ef35ea9664 is 50, key is test_row_0/B:col10/1731631635624/Put/seqid=0 2024-11-15T00:47:15,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742416_1592 (size=9657) 2024-11-15T00:47:15,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/f2f96ac7ae344160ab9267ef35ea9664 2024-11-15T00:47:15,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/5423db67d93547a290ea8ba9fcf73c8e is 50, key is test_row_0/C:col10/1731631635624/Put/seqid=0 2024-11-15T00:47:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631695800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631695800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631695801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:15,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742417_1593 (size=9657) 2024-11-15T00:47:15,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/5423db67d93547a290ea8ba9fcf73c8e 2024-11-15T00:47:15,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/2e082444302645c797fe05148e744b29 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/2e082444302645c797fe05148e744b29 2024-11-15T00:47:15,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/2e082444302645c797fe05148e744b29, entries=200, sequenceid=64, filesize=38.6 K 2024-11-15T00:47:15,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/f2f96ac7ae344160ab9267ef35ea9664 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f2f96ac7ae344160ab9267ef35ea9664 2024-11-15T00:47:15,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f2f96ac7ae344160ab9267ef35ea9664, entries=100, sequenceid=64, filesize=9.4 K 2024-11-15T00:47:15,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/5423db67d93547a290ea8ba9fcf73c8e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/5423db67d93547a290ea8ba9fcf73c8e 2024-11-15T00:47:15,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/5423db67d93547a290ea8ba9fcf73c8e, entries=100, sequenceid=64, filesize=9.4 K 2024-11-15T00:47:15,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 244ms, sequenceid=64, compaction requested=true 2024-11-15T00:47:15,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:15,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:15,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:15,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:47:15,869 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:15,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:15,869 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:15,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:47:15,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:15,869 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:15,869 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/B is initiating minor compaction (all files) 2024-11-15T00:47:15,869 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/B in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:15,870 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/3a499b6902894fffbe00dad0998fe683, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/c8799593c5e4436e95140eb627c67e72, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f2f96ac7ae344160ab9267ef35ea9664] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=32.9 K 2024-11-15T00:47:15,870 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:15,870 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/A is initiating minor compaction (all files) 2024-11-15T00:47:15,870 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/A in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:15,870 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/22638295d9d14264941a09fb892a63cb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0d7cdac77b914e529903734c8e15316b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/2e082444302645c797fe05148e744b29] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=107.5 K 2024-11-15T00:47:15,870 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:15,870 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/22638295d9d14264941a09fb892a63cb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0d7cdac77b914e529903734c8e15316b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/2e082444302645c797fe05148e744b29] 2024-11-15T00:47:15,871 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a499b6902894fffbe00dad0998fe683, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731631633060 2024-11-15T00:47:15,871 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22638295d9d14264941a09fb892a63cb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731631633060 2024-11-15T00:47:15,871 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting c8799593c5e4436e95140eb627c67e72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731631633154 2024-11-15T00:47:15,871 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d7cdac77b914e529903734c8e15316b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731631633154 2024-11-15T00:47:15,872 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e082444302645c797fe05148e744b29, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731631635292 2024-11-15T00:47:15,872 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f96ac7ae344160ab9267ef35ea9664, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731631635623 2024-11-15T00:47:15,887 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:15,888 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/70a5dddceddb4d949b4bd5b39630ef27 is 50, key is test_row_0/B:col10/1731631635624/Put/seqid=0 2024-11-15T00:47:15,895 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:15,921 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115a9b038203a0d4ffab186462e867802c9_8072ff2efb6653ab0fbbcdf205cf0e99 store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:15,923 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115a9b038203a0d4ffab186462e867802c9_8072ff2efb6653ab0fbbcdf205cf0e99, store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:15,923 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a9b038203a0d4ffab186462e867802c9_8072ff2efb6653ab0fbbcdf205cf0e99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:15,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742418_1594 (size=12104) 2024-11-15T00:47:15,964 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/70a5dddceddb4d949b4bd5b39630ef27 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/70a5dddceddb4d949b4bd5b39630ef27 2024-11-15T00:47:15,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742419_1595 (size=4469) 2024-11-15T00:47:15,978 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/B of 8072ff2efb6653ab0fbbcdf205cf0e99 into 70a5dddceddb4d949b4bd5b39630ef27(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:15,978 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:15,978 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/B, priority=13, startTime=1731631635868; duration=0sec 2024-11-15T00:47:15,978 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:15,978 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:B 2024-11-15T00:47:15,978 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:15,979 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:15,979 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/C is initiating minor compaction (all files) 2024-11-15T00:47:15,979 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/C in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:15,979 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/01e40590fd854a6da7ae9d410ce5a2c6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/fdca5f98b3ee40aa90590d4998803df4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/5423db67d93547a290ea8ba9fcf73c8e] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=32.9 K 2024-11-15T00:47:15,979 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e40590fd854a6da7ae9d410ce5a2c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731631633060 2024-11-15T00:47:15,979 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting fdca5f98b3ee40aa90590d4998803df4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731631633154 2024-11-15T00:47:15,980 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 5423db67d93547a290ea8ba9fcf73c8e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731631635623 2024-11-15T00:47:15,995 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#C#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:15,995 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/659ccc59a8d2494c8ba5a0e0a5b4de3a is 50, key is test_row_0/C:col10/1731631635624/Put/seqid=0 2024-11-15T00:47:16,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742420_1596 (size=12104) 2024-11-15T00:47:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:16,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:47:16,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:16,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:16,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:16,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:16,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:16,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:16,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bf3e26395fb0488c9a7710b89535456c_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631636010/Put/seqid=0 2024-11-15T00:47:16,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631696034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631696036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631696037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742421_1597 (size=12154) 2024-11-15T00:47:16,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631696149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631696149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631696149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631696361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631696362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631696362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,371 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#A#compaction#504 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:16,371 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/d137db0de05e485fb89c6d7f4920c3e4 is 175, key is test_row_0/A:col10/1731631635624/Put/seqid=0 2024-11-15T00:47:16,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742422_1598 (size=31165) 2024-11-15T00:47:16,433 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/659ccc59a8d2494c8ba5a0e0a5b4de3a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/659ccc59a8d2494c8ba5a0e0a5b4de3a 2024-11-15T00:47:16,437 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/C of 8072ff2efb6653ab0fbbcdf205cf0e99 into 659ccc59a8d2494c8ba5a0e0a5b4de3a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:16,438 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:16,438 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/C, priority=13, startTime=1731631635869; duration=0sec 2024-11-15T00:47:16,438 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:16,438 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:C 2024-11-15T00:47:16,446 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:16,450 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bf3e26395fb0488c9a7710b89535456c_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf3e26395fb0488c9a7710b89535456c_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:16,450 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/5b856442525a4fba9ecd9b9cc07da686, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:16,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/5b856442525a4fba9ecd9b9cc07da686 is 175, key is test_row_0/A:col10/1731631636010/Put/seqid=0 2024-11-15T00:47:16,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742423_1599 (size=30955) 2024-11-15T00:47:16,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631696671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631696679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631696680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:16,804 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/d137db0de05e485fb89c6d7f4920c3e4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/d137db0de05e485fb89c6d7f4920c3e4 2024-11-15T00:47:16,810 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/A of 8072ff2efb6653ab0fbbcdf205cf0e99 into d137db0de05e485fb89c6d7f4920c3e4(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:16,810 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:16,810 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/A, priority=13, startTime=1731631635868; duration=0sec 2024-11-15T00:47:16,810 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:16,810 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:A 2024-11-15T00:47:16,893 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/5b856442525a4fba9ecd9b9cc07da686 2024-11-15T00:47:16,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/8673079268b44555a8a2395e1e96eb32 is 50, key is test_row_0/B:col10/1731631636010/Put/seqid=0 2024-11-15T00:47:16,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742424_1600 (size=12001) 2024-11-15T00:47:17,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631697189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:17,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631697190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:17,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631697189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:17,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:17,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:17,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T00:47:17,209 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:17,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:47:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-15T00:47:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T00:47:17,213 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:47:17,215 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:47:17,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:47:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T00:47:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631697340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:17,342 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in 8072ff2efb6653ab0fbbcdf205cf0e99 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:47:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:17,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/8673079268b44555a8a2395e1e96eb32 2024-11-15T00:47:17,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631697350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:17,354 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in 8072ff2efb6653ab0fbbcdf205cf0e99 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:47:17,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:17,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:17,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:17,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-15T00:47:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:17,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:17,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/ef59e66eb7284dae86cc7828276f6df6 is 50, key is test_row_0/C:col10/1731631636010/Put/seqid=0 2024-11-15T00:47:17,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742425_1601 (size=12001) 2024-11-15T00:47:17,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/ef59e66eb7284dae86cc7828276f6df6 2024-11-15T00:47:17,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/5b856442525a4fba9ecd9b9cc07da686 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/5b856442525a4fba9ecd9b9cc07da686 2024-11-15T00:47:17,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/5b856442525a4fba9ecd9b9cc07da686, entries=150, sequenceid=85, filesize=30.2 K 2024-11-15T00:47:17,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/8673079268b44555a8a2395e1e96eb32 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8673079268b44555a8a2395e1e96eb32 2024-11-15T00:47:17,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8673079268b44555a8a2395e1e96eb32, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T00:47:17,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/ef59e66eb7284dae86cc7828276f6df6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ef59e66eb7284dae86cc7828276f6df6 2024-11-15T00:47:17,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ef59e66eb7284dae86cc7828276f6df6, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T00:47:17,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1484ms, sequenceid=85, compaction requested=false 2024-11-15T00:47:17,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:17,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T00:47:17,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-15T00:47:17,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:17,529 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:47:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:17,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cf36ce9d264e432c8ef90202c0af0bcd_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631636035/Put/seqid=0 2024-11-15T00:47:17,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742426_1602 (size=12154) 2024-11-15T00:47:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T00:47:18,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:18,052 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cf36ce9d264e432c8ef90202c0af0bcd_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cf36ce9d264e432c8ef90202c0af0bcd_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:18,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/0897c85882844c2a9557d9cf49ec8364, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:18,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/0897c85882844c2a9557d9cf49ec8364 is 175, key is test_row_0/A:col10/1731631636035/Put/seqid=0 2024-11-15T00:47:18,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742427_1603 (size=30955) 2024-11-15T00:47:18,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:18,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631698270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631698270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631698278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T00:47:18,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631698379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631698389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631698390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,491 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/0897c85882844c2a9557d9cf49ec8364 2024-11-15T00:47:18,500 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:47:18,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/86be62d9de084e1cb561ef89c4e725fc is 50, key is test_row_0/B:col10/1731631636035/Put/seqid=0 2024-11-15T00:47:18,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742428_1604 (size=12001) 2024-11-15T00:47:18,532 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/86be62d9de084e1cb561ef89c4e725fc 2024-11-15T00:47:18,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/7e2e1f883f984bff982d5cda21c1ba2d is 50, key is test_row_0/C:col10/1731631636035/Put/seqid=0 2024-11-15T00:47:18,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742429_1605 (size=12001) 2024-11-15T00:47:18,582 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/7e2e1f883f984bff982d5cda21c1ba2d 2024-11-15T00:47:18,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631698589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631698600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631698604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/0897c85882844c2a9557d9cf49ec8364 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0897c85882844c2a9557d9cf49ec8364 2024-11-15T00:47:18,642 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0897c85882844c2a9557d9cf49ec8364, entries=150, sequenceid=99, filesize=30.2 K 2024-11-15T00:47:18,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/86be62d9de084e1cb561ef89c4e725fc as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/86be62d9de084e1cb561ef89c4e725fc 2024-11-15T00:47:18,657 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/86be62d9de084e1cb561ef89c4e725fc, entries=150, sequenceid=99, filesize=11.7 K 2024-11-15T00:47:18,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/7e2e1f883f984bff982d5cda21c1ba2d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/7e2e1f883f984bff982d5cda21c1ba2d 2024-11-15T00:47:18,665 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/7e2e1f883f984bff982d5cda21c1ba2d, entries=150, sequenceid=99, filesize=11.7 K 2024-11-15T00:47:18,668 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1139ms, sequenceid=99, compaction requested=true 2024-11-15T00:47:18,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:18,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:18,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-15T00:47:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-15T00:47:18,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-15T00:47:18,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4550 sec 2024-11-15T00:47:18,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.4650 sec 2024-11-15T00:47:18,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T00:47:18,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:18,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:18,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:18,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:18,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:18,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:18,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e753c0ddbf254f0a825f1c5423d13df2_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631638267/Put/seqid=0 2024-11-15T00:47:18,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742430_1606 (size=14594) 2024-11-15T00:47:18,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631698952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631698957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:18,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631698961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:19,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631699069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:19,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631699069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:19,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631699079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:19,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631699279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:19,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631699279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:19,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631699289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:19,348 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:19,351 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e753c0ddbf254f0a825f1c5423d13df2_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e753c0ddbf254f0a825f1c5423d13df2_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:19,352 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/409e3a6ab55b4f4ea00509b60d03452f, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:19,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/409e3a6ab55b4f4ea00509b60d03452f is 175, key is test_row_0/A:col10/1731631638267/Put/seqid=0 2024-11-15T00:47:19,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742431_1607 (size=39549) 2024-11-15T00:47:19,356 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/409e3a6ab55b4f4ea00509b60d03452f 2024-11-15T00:47:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-15T00:47:19,359 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:19,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:47:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-15T00:47:19,362 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:47:19,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/47548f2e5b8f42cf9decf6391ada48ea is 50, key is test_row_0/B:col10/1731631638267/Put/seqid=0 2024-11-15T00:47:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T00:47:19,362 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:47:19,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:47:19,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742432_1608 (size=12001) 2024-11-15T00:47:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T00:47:19,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T00:47:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:19,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631699589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:19,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631699589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631699599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:19,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T00:47:19,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:19,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:19,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:19,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T00:47:19,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/47548f2e5b8f42cf9decf6391ada48ea 2024-11-15T00:47:19,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/4459a81b1b76428c9d0f2e706f51f0d9 is 50, key is test_row_0/C:col10/1731631638267/Put/seqid=0 2024-11-15T00:47:19,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742433_1609 (size=12001) 2024-11-15T00:47:19,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T00:47:19,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:19,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:19,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:19,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T00:47:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:19,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T00:47:20,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631700099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:20,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631700109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:20,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631700109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:20,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T00:47:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:20,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:20,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/4459a81b1b76428c9d0f2e706f51f0d9 2024-11-15T00:47:20,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/409e3a6ab55b4f4ea00509b60d03452f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/409e3a6ab55b4f4ea00509b60d03452f 2024-11-15T00:47:20,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/409e3a6ab55b4f4ea00509b60d03452f, entries=200, sequenceid=120, filesize=38.6 K 2024-11-15T00:47:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/47548f2e5b8f42cf9decf6391ada48ea as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/47548f2e5b8f42cf9decf6391ada48ea 2024-11-15T00:47:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/47548f2e5b8f42cf9decf6391ada48ea, entries=150, sequenceid=120, filesize=11.7 K 2024-11-15T00:47:20,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/4459a81b1b76428c9d0f2e706f51f0d9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4459a81b1b76428c9d0f2e706f51f0d9 2024-11-15T00:47:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4459a81b1b76428c9d0f2e706f51f0d9, entries=150, sequenceid=120, filesize=11.7 K 2024-11-15T00:47:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1288ms, sequenceid=120, compaction requested=true 2024-11-15T00:47:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:20,201 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:47:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:47:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:47:20,201 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:47:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,202 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:47:20,202 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132624 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:47:20,202 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/B is initiating minor compaction (all files) 2024-11-15T00:47:20,202 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/A is initiating minor compaction (all files) 2024-11-15T00:47:20,202 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/A in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:20,202 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/B in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,202 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/d137db0de05e485fb89c6d7f4920c3e4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/5b856442525a4fba9ecd9b9cc07da686, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0897c85882844c2a9557d9cf49ec8364, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/409e3a6ab55b4f4ea00509b60d03452f] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=129.5 K 2024-11-15T00:47:20,202 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/70a5dddceddb4d949b4bd5b39630ef27, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8673079268b44555a8a2395e1e96eb32, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/86be62d9de084e1cb561ef89c4e725fc, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/47548f2e5b8f42cf9decf6391ada48ea] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=47.0 K 2024-11-15T00:47:20,202 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:20,202 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/d137db0de05e485fb89c6d7f4920c3e4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/5b856442525a4fba9ecd9b9cc07da686, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0897c85882844c2a9557d9cf49ec8364, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/409e3a6ab55b4f4ea00509b60d03452f] 2024-11-15T00:47:20,203 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a5dddceddb4d949b4bd5b39630ef27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731631633158 2024-11-15T00:47:20,203 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting d137db0de05e485fb89c6d7f4920c3e4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731631633158 2024-11-15T00:47:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,203 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 8673079268b44555a8a2395e1e96eb32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731631635672 2024-11-15T00:47:20,203 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b856442525a4fba9ecd9b9cc07da686, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731631635672 2024-11-15T00:47:20,203 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0897c85882844c2a9557d9cf49ec8364, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731631636028 2024-11-15T00:47:20,203 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 86be62d9de084e1cb561ef89c4e725fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731631636028 2024-11-15T00:47:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,203 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 409e3a6ab55b4f4ea00509b60d03452f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731631638240 2024-11-15T00:47:20,203 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 47548f2e5b8f42cf9decf6391ada48ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731631638267 2024-11-15T00:47:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,208 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:20,209 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#B#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:20,209 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/8c804ffb9dc043059d762d75713d6007 is 50, key is test_row_0/B:col10/1731631638267/Put/seqid=0 2024-11-15T00:47:20,210 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411157df1231d74064eada09f1acf9788f710_8072ff2efb6653ab0fbbcdf205cf0e99 store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:20,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742434_1610 (size=12241) 2024-11-15T00:47:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,212 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411157df1231d74064eada09f1acf9788f710_8072ff2efb6653ab0fbbcdf205cf0e99, store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:20,213 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157df1231d74064eada09f1acf9788f710_8072ff2efb6653ab0fbbcdf205cf0e99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742435_1611 (size=4469) 2024-11-15T00:47:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-15T00:47:20,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,290 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T00:47:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159da851d10852421a82d3b973726d7944_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631638958/Put/seqid=0 2024-11-15T00:47:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742436_1612 (size=9814) 2024-11-15T00:47:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,328 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159da851d10852421a82d3b973726d7944_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159da851d10852421a82d3b973726d7944_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/7f6eceae915144eea474102a7edea122, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:20,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/7f6eceae915144eea474102a7edea122 is 175, key is test_row_0/A:col10/1731631638958/Put/seqid=0 2024-11-15T00:47:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742437_1613 (size=22461) 2024-11-15T00:47:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T00:47:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,618 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/8c804ffb9dc043059d762d75713d6007 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8c804ffb9dc043059d762d75713d6007 2024-11-15T00:47:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,621 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/B of 8072ff2efb6653ab0fbbcdf205cf0e99 into 8c804ffb9dc043059d762d75713d6007(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:20,621 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:20,622 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/B, priority=12, startTime=1731631640201; duration=0sec 2024-11-15T00:47:20,622 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:20,622 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:B 2024-11-15T00:47:20,622 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:47:20,622 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#A#compaction#516 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,623 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/89e08a01ad23447b99608e66e4b2f55d is 175, key is test_row_0/A:col10/1731631638267/Put/seqid=0 2024-11-15T00:47:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,624 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:47:20,624 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/C is initiating minor compaction (all files) 2024-11-15T00:47:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,624 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/C in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:20,624 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/659ccc59a8d2494c8ba5a0e0a5b4de3a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ef59e66eb7284dae86cc7828276f6df6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/7e2e1f883f984bff982d5cda21c1ba2d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4459a81b1b76428c9d0f2e706f51f0d9] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=47.0 K 2024-11-15T00:47:20,625 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 659ccc59a8d2494c8ba5a0e0a5b4de3a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731631633158 2024-11-15T00:47:20,625 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ef59e66eb7284dae86cc7828276f6df6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731631635672 2024-11-15T00:47:20,625 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e2e1f883f984bff982d5cda21c1ba2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731631636028 2024-11-15T00:47:20,626 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 4459a81b1b76428c9d0f2e706f51f0d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731631638267 2024-11-15T00:47:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742438_1614 (size=31195) 2024-11-15T00:47:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,630 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/89e08a01ad23447b99608e66e4b2f55d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/89e08a01ad23447b99608e66e4b2f55d 2024-11-15T00:47:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,633 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#C#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,633 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/ebdf68e7f4b4424999604a1dd7b138bd is 50, key is test_row_0/C:col10/1731631638267/Put/seqid=0 2024-11-15T00:47:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,634 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/A of 8072ff2efb6653ab0fbbcdf205cf0e99 into 89e08a01ad23447b99608e66e4b2f55d(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:20,634 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:20,634 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/A, priority=12, startTime=1731631640201; duration=0sec 2024-11-15T00:47:20,634 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:20,634 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:A 2024-11-15T00:47:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742439_1615 (size=12241) 2024-11-15T00:47:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,747 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/7f6eceae915144eea474102a7edea122 2024-11-15T00:47:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/1fda559ce2eb4930a4897f8d29ac1a80 is 50, key is test_row_0/B:col10/1731631638958/Put/seqid=0 2024-11-15T00:47:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742440_1616 (size=9757) 2024-11-15T00:47:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,053 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/ebdf68e7f4b4424999604a1dd7b138bd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ebdf68e7f4b4424999604a1dd7b138bd 2024-11-15T00:47:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,057 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/C of 8072ff2efb6653ab0fbbcdf205cf0e99 into ebdf68e7f4b4424999604a1dd7b138bd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:21,057 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:21,057 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/C, priority=12, startTime=1731631640201; duration=0sec 2024-11-15T00:47:21,058 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:21,058 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:C 2024-11-15T00:47:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:21,156 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/1fda559ce2eb4930a4897f8d29ac1a80 2024-11-15T00:47:21,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631701162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731631641152, max timestamp=1731631641166Num uniques 859; 2024-11-15T00:47:21,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T00:47:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/ba9f83175a884a788757592ff37a286c is 50, key is test_row_0/C:col10/1731631638958/Put/seqid=0 2024-11-15T00:47:21,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742441_1617 (size=9757) 2024-11-15T00:47:21,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631701183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631701187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631701287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631701299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631701299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631701350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,354 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_1' in 8072ff2efb6653ab0fbbcdf205cf0e99 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:47:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631701359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,361 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in 8072ff2efb6653ab0fbbcdf205cf0e99 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:47:21,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631701500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T00:47:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631701509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631701509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,580 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/ba9f83175a884a788757592ff37a286c 2024-11-15T00:47:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/7f6eceae915144eea474102a7edea122 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/7f6eceae915144eea474102a7edea122 2024-11-15T00:47:21,587 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/7f6eceae915144eea474102a7edea122, entries=100, sequenceid=131, filesize=21.9 K 2024-11-15T00:47:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/1fda559ce2eb4930a4897f8d29ac1a80 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/1fda559ce2eb4930a4897f8d29ac1a80 2024-11-15T00:47:21,590 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/1fda559ce2eb4930a4897f8d29ac1a80, entries=100, sequenceid=131, filesize=9.5 K 2024-11-15T00:47:21,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/ba9f83175a884a788757592ff37a286c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ba9f83175a884a788757592ff37a286c 2024-11-15T00:47:21,594 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ba9f83175a884a788757592ff37a286c, entries=100, sequenceid=131, filesize=9.5 K 2024-11-15T00:47:21,595 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1305ms, sequenceid=131, compaction requested=false 2024-11-15T00:47:21,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:21,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:21,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-15T00:47:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-15T00:47:21,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-15T00:47:21,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2340 sec 2024-11-15T00:47:21,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.2390 sec 2024-11-15T00:47:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:21,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=131.94 KB heapSize=386.72 KB 2024-11-15T00:47:21,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:21,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:21,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:21,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:21,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:21,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111525512ade8fbc4c8db357f3869f97184f_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631641185/Put/seqid=0 2024-11-15T00:47:21,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742442_1618 (size=14794) 2024-11-15T00:47:21,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631701826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631701829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631701830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631701939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631701940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631701940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:22,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631702149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631702149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:22,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631702149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:22,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:22,223 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111525512ade8fbc4c8db357f3869f97184f_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111525512ade8fbc4c8db357f3869f97184f_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:22,224 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/b7d3cbc765584a0888f01a80113367c6, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:22,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/b7d3cbc765584a0888f01a80113367c6 is 175, key is test_row_0/A:col10/1731631641185/Put/seqid=0 2024-11-15T00:47:22,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742443_1619 (size=39749) 2024-11-15T00:47:22,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/b7d3cbc765584a0888f01a80113367c6 2024-11-15T00:47:22,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/069d68476b374454b22e8f526d007a5a is 50, key is test_row_0/B:col10/1731631641185/Put/seqid=0 2024-11-15T00:47:22,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742444_1620 (size=12151) 2024-11-15T00:47:22,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631702459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631702460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631702460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:22,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/069d68476b374454b22e8f526d007a5a 2024-11-15T00:47:22,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/099574a236ee41ef98eaa7c1dad77ef1 is 50, key is test_row_0/C:col10/1731631641185/Put/seqid=0 2024-11-15T00:47:22,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742445_1621 (size=12151) 2024-11-15T00:47:22,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/099574a236ee41ef98eaa7c1dad77ef1 2024-11-15T00:47:22,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/b7d3cbc765584a0888f01a80113367c6 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b7d3cbc765584a0888f01a80113367c6 2024-11-15T00:47:22,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b7d3cbc765584a0888f01a80113367c6, entries=200, sequenceid=161, filesize=38.8 K 2024-11-15T00:47:22,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/069d68476b374454b22e8f526d007a5a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/069d68476b374454b22e8f526d007a5a 2024-11-15T00:47:22,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/069d68476b374454b22e8f526d007a5a, entries=150, sequenceid=161, filesize=11.9 K 2024-11-15T00:47:22,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/099574a236ee41ef98eaa7c1dad77ef1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/099574a236ee41ef98eaa7c1dad77ef1 2024-11-15T00:47:22,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/099574a236ee41ef98eaa7c1dad77ef1, entries=150, sequenceid=161, filesize=11.9 K 2024-11-15T00:47:22,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~386.67 KB/395952, currentSize=46.96 KB/48090 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 946ms, sequenceid=161, compaction requested=true 2024-11-15T00:47:22,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:22,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:22,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:22,757 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:22,757 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:22,759 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:22,759 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:22,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:47:22,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:22,759 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/B is initiating minor compaction (all files) 2024-11-15T00:47:22,759 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/A is initiating minor compaction (all files) 2024-11-15T00:47:22,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:47:22,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:22,759 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/A in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:22,759 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/89e08a01ad23447b99608e66e4b2f55d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/7f6eceae915144eea474102a7edea122, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b7d3cbc765584a0888f01a80113367c6] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=91.2 K 2024-11-15T00:47:22,759 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/B in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:22,759 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:22,759 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/89e08a01ad23447b99608e66e4b2f55d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/7f6eceae915144eea474102a7edea122, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b7d3cbc765584a0888f01a80113367c6] 2024-11-15T00:47:22,759 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8c804ffb9dc043059d762d75713d6007, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/1fda559ce2eb4930a4897f8d29ac1a80, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/069d68476b374454b22e8f526d007a5a] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=33.3 K 2024-11-15T00:47:22,760 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e08a01ad23447b99608e66e4b2f55d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731631638267 2024-11-15T00:47:22,760 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c804ffb9dc043059d762d75713d6007, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731631638267 2024-11-15T00:47:22,760 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f6eceae915144eea474102a7edea122, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731631638946 2024-11-15T00:47:22,760 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fda559ce2eb4930a4897f8d29ac1a80, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731631638946 2024-11-15T00:47:22,761 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 069d68476b374454b22e8f526d007a5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731631641175 2024-11-15T00:47:22,761 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7d3cbc765584a0888f01a80113367c6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731631641175 2024-11-15T00:47:22,768 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:22,769 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#B#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:22,770 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/f58c66ad0f9748d98230adf2e9b0b018 is 50, key is test_row_0/B:col10/1731631641185/Put/seqid=0 2024-11-15T00:47:22,779 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411152ce173514cb54d1583a1e3df2c7b7d61_8072ff2efb6653ab0fbbcdf205cf0e99 store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:22,780 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411152ce173514cb54d1583a1e3df2c7b7d61_8072ff2efb6653ab0fbbcdf205cf0e99, store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:22,781 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152ce173514cb54d1583a1e3df2c7b7d61_8072ff2efb6653ab0fbbcdf205cf0e99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:22,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742446_1622 (size=12493) 2024-11-15T00:47:22,827 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/f58c66ad0f9748d98230adf2e9b0b018 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f58c66ad0f9748d98230adf2e9b0b018 2024-11-15T00:47:22,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742447_1623 (size=4469) 2024-11-15T00:47:22,830 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#A#compaction#524 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:22,831 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/b2d3bce07cc543b9b28e25d7df925ea9 is 175, key is test_row_0/A:col10/1731631641185/Put/seqid=0 2024-11-15T00:47:22,834 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/B of 8072ff2efb6653ab0fbbcdf205cf0e99 into f58c66ad0f9748d98230adf2e9b0b018(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:22,835 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:22,835 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/B, priority=13, startTime=1731631642757; duration=0sec 2024-11-15T00:47:22,835 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:22,835 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:B 2024-11-15T00:47:22,835 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:22,836 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:22,836 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/C is initiating minor compaction (all files) 2024-11-15T00:47:22,836 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/C in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:22,836 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ebdf68e7f4b4424999604a1dd7b138bd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ba9f83175a884a788757592ff37a286c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/099574a236ee41ef98eaa7c1dad77ef1] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=33.3 K 2024-11-15T00:47:22,836 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ebdf68e7f4b4424999604a1dd7b138bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731631638267 2024-11-15T00:47:22,836 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9f83175a884a788757592ff37a286c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731631638946 2024-11-15T00:47:22,837 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 099574a236ee41ef98eaa7c1dad77ef1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731631641175 2024-11-15T00:47:22,866 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#C#compaction#526 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:22,866 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/db3b2c4ebe1a4193adc7652f69835e56 is 50, key is test_row_0/C:col10/1731631641185/Put/seqid=0 2024-11-15T00:47:22,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742448_1624 (size=31447) 2024-11-15T00:47:22,879 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/b2d3bce07cc543b9b28e25d7df925ea9 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2d3bce07cc543b9b28e25d7df925ea9 2024-11-15T00:47:22,892 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/A of 8072ff2efb6653ab0fbbcdf205cf0e99 into b2d3bce07cc543b9b28e25d7df925ea9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:22,892 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:22,892 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/A, priority=13, startTime=1731631642757; duration=0sec 2024-11-15T00:47:22,893 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:22,893 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:A 2024-11-15T00:47:22,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742449_1625 (size=12493) 2024-11-15T00:47:22,924 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/db3b2c4ebe1a4193adc7652f69835e56 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/db3b2c4ebe1a4193adc7652f69835e56 2024-11-15T00:47:22,930 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/C of 8072ff2efb6653ab0fbbcdf205cf0e99 into db3b2c4ebe1a4193adc7652f69835e56(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:22,930 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:22,931 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/C, priority=13, startTime=1731631642759; duration=0sec 2024-11-15T00:47:22,931 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:22,931 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:C 2024-11-15T00:47:22,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T00:47:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:22,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:22,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:22,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:22,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:22,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:22,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:22,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157c919d79d846477dbd15ab4bca25525c_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631641829/Put/seqid=0 2024-11-15T00:47:23,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742450_1626 (size=14794) 2024-11-15T00:47:23,017 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:23,040 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157c919d79d846477dbd15ab4bca25525c_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c919d79d846477dbd15ab4bca25525c_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:23,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631703033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,044 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/b2b9d412fef84da0a21312af9714e127, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:23,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631643018, max timestamp=1731631643031Num uniques 859; 2024-11-15T00:47:23,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:23,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/b2b9d412fef84da0a21312af9714e127 is 175, key is test_row_0/A:col10/1731631641829/Put/seqid=0 2024-11-15T00:47:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631703032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631643018, max timestamp=1731631643031Num uniques 859; 2024-11-15T00:47:23,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:23,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731631643018, max timestamp=1731631643031Num uniques 859; 2024-11-15T00:47:23,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T00:47:23,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742451_1627 (size=39749) 2024-11-15T00:47:23,088 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/b2b9d412fef84da0a21312af9714e127 2024-11-15T00:47:23,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/a03c6fda8da743289ec5a60dec90adf5 is 50, key is test_row_0/B:col10/1731631641829/Put/seqid=0 2024-11-15T00:47:23,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742452_1628 (size=12151) 2024-11-15T00:47:23,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631703166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631703169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631703170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631703284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631703289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631703289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631703499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631703499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631703500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T00:47:23,529 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:23,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:47:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-15T00:47:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:47:23,531 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:47:23,532 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:47:23,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:47:23,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/a03c6fda8da743289ec5a60dec90adf5 2024-11-15T00:47:23,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/bbc117530fc54a3e9d2e1c7cb0fb2916 is 50, key is test_row_0/C:col10/1731631641829/Put/seqid=0 2024-11-15T00:47:23,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742453_1629 (size=12151) 2024-11-15T00:47:23,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/bbc117530fc54a3e9d2e1c7cb0fb2916 2024-11-15T00:47:23,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/b2b9d412fef84da0a21312af9714e127 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2b9d412fef84da0a21312af9714e127 2024-11-15T00:47:23,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2b9d412fef84da0a21312af9714e127, entries=200, sequenceid=176, filesize=38.8 K 2024-11-15T00:47:23,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/a03c6fda8da743289ec5a60dec90adf5 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/a03c6fda8da743289ec5a60dec90adf5 2024-11-15T00:47:23,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/a03c6fda8da743289ec5a60dec90adf5, entries=150, sequenceid=176, filesize=11.9 K 2024-11-15T00:47:23,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/bbc117530fc54a3e9d2e1c7cb0fb2916 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/bbc117530fc54a3e9d2e1c7cb0fb2916 2024-11-15T00:47:23,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/bbc117530fc54a3e9d2e1c7cb0fb2916, entries=150, sequenceid=176, filesize=11.9 K 2024-11-15T00:47:23,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.37 KB/123255 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 631ms, sequenceid=176, compaction requested=false 2024-11-15T00:47:23,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:47:23,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-15T00:47:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:23,685 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=120.37 KB heapSize=344.23 KB 2024-11-15T00:47:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:23,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c671f2b9409f481a9ce1f07f9d274013_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631643132/Put/seqid=0 2024-11-15T00:47:23,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742454_1630 (size=12304) 2024-11-15T00:47:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:23,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:23,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631703848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631703851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631703852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:47:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631703961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631703969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631703972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:23,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:24,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:47:24,182 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c671f2b9409f481a9ce1f07f9d274013_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c671f2b9409f481a9ce1f07f9d274013_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:24,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:24,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e is 175, key is test_row_0/A:col10/1731631643132/Put/seqid=0 2024-11-15T00:47:24,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631704180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:24,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631704181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:24,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631704196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:24,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742455_1631 (size=31105) 2024-11-15T00:47:24,214 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e 2024-11-15T00:47:24,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/038af211ed5549c7b054063fc845975a is 50, key is test_row_0/B:col10/1731631643132/Put/seqid=0 2024-11-15T00:47:24,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742456_1632 (size=12151) 2024-11-15T00:47:24,250 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/038af211ed5549c7b054063fc845975a 2024-11-15T00:47:24,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/9c1b1003c3664c09a1826a945d4e79a3 is 50, key is test_row_0/C:col10/1731631643132/Put/seqid=0 2024-11-15T00:47:24,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742457_1633 (size=12151) 2024-11-15T00:47:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631704489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:24,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631704499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:24,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631704509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:24,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:24,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:47:24,720 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/9c1b1003c3664c09a1826a945d4e79a3 2024-11-15T00:47:24,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e 2024-11-15T00:47:24,768 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e, entries=150, sequenceid=211, filesize=30.4 K 2024-11-15T00:47:24,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/038af211ed5549c7b054063fc845975a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/038af211ed5549c7b054063fc845975a 2024-11-15T00:47:24,773 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/038af211ed5549c7b054063fc845975a, entries=150, sequenceid=211, filesize=11.9 K 2024-11-15T00:47:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/9c1b1003c3664c09a1826a945d4e79a3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/9c1b1003c3664c09a1826a945d4e79a3 2024-11-15T00:47:24,785 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/9c1b1003c3664c09a1826a945d4e79a3, entries=150, sequenceid=211, filesize=11.9 K 2024-11-15T00:47:24,787 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~120.37 KB/123255, heapSize ~344.18 KB/352440, currentSize=60.38 KB/61830 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1101ms, sequenceid=211, compaction requested=true 2024-11-15T00:47:24,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:24,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:24,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-15T00:47:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-15T00:47:24,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-15T00:47:24,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2680 sec 2024-11-15T00:47:24,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.2820 sec 2024-11-15T00:47:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:25,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T00:47:25,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:25,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:25,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:25,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:25,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:25,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:25,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c644bca86e714bb1813fe640cbed9bb5_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631643837/Put/seqid=0 2024-11-15T00:47:25,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742458_1634 (size=14794) 2024-11-15T00:47:25,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631705111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631705111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631705112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631705219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631705229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631705229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631705429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631705439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631705439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,449 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:25,451 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c644bca86e714bb1813fe640cbed9bb5_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c644bca86e714bb1813fe640cbed9bb5_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:25,452 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/14ae39b0e0cd41d09632e4c3c922a05d, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:25,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/14ae39b0e0cd41d09632e4c3c922a05d is 175, key is test_row_0/A:col10/1731631643837/Put/seqid=0 2024-11-15T00:47:25,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742459_1635 (size=39749) 2024-11-15T00:47:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:47:25,699 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:25,700 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:47:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-15T00:47:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T00:47:25,702 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:47:25,703 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:47:25,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:47:25,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631705739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631705752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631705752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T00:47:25,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T00:47:25,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:25,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:25,856 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/14ae39b0e0cd41d09632e4c3c922a05d 2024-11-15T00:47:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:25,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/2d1369f738ec441eb5e6bb527babad7f is 50, key is test_row_0/B:col10/1731631643837/Put/seqid=0 2024-11-15T00:47:25,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742460_1636 (size=12151) 2024-11-15T00:47:25,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/2d1369f738ec441eb5e6bb527babad7f 2024-11-15T00:47:25,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/f23def3431f145a4897f0207611ffc46 is 50, key is test_row_0/C:col10/1731631643837/Put/seqid=0 2024-11-15T00:47:25,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742461_1637 (size=12151) 2024-11-15T00:47:25,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/f23def3431f145a4897f0207611ffc46 2024-11-15T00:47:25,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/14ae39b0e0cd41d09632e4c3c922a05d as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/14ae39b0e0cd41d09632e4c3c922a05d 2024-11-15T00:47:25,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/14ae39b0e0cd41d09632e4c3c922a05d, entries=200, sequenceid=224, filesize=38.8 K 2024-11-15T00:47:25,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/2d1369f738ec441eb5e6bb527babad7f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2d1369f738ec441eb5e6bb527babad7f 2024-11-15T00:47:25,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2d1369f738ec441eb5e6bb527babad7f, entries=150, sequenceid=224, filesize=11.9 K 2024-11-15T00:47:25,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/f23def3431f145a4897f0207611ffc46 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/f23def3431f145a4897f0207611ffc46 2024-11-15T00:47:25,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/f23def3431f145a4897f0207611ffc46, entries=150, sequenceid=224, filesize=11.9 K 2024-11-15T00:47:25,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 938ms, sequenceid=224, compaction requested=true 2024-11-15T00:47:25,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:25,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:25,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:25,947 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:47:25,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:47:25,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:25,947 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:47:25,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:47:25,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:25,948 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:47:25,948 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/A is initiating minor compaction (all files) 2024-11-15T00:47:25,949 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/A in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:25,949 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2d3bce07cc543b9b28e25d7df925ea9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2b9d412fef84da0a21312af9714e127, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/14ae39b0e0cd41d09632e4c3c922a05d] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=138.7 K 2024-11-15T00:47:25,949 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:25,949 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2d3bce07cc543b9b28e25d7df925ea9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2b9d412fef84da0a21312af9714e127, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/14ae39b0e0cd41d09632e4c3c922a05d] 2024-11-15T00:47:25,949 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:47:25,949 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/B is initiating minor compaction (all files) 2024-11-15T00:47:25,949 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/B in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:25,949 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f58c66ad0f9748d98230adf2e9b0b018, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/a03c6fda8da743289ec5a60dec90adf5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/038af211ed5549c7b054063fc845975a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2d1369f738ec441eb5e6bb527babad7f] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=47.8 K 2024-11-15T00:47:25,949 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d3bce07cc543b9b28e25d7df925ea9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731631641175 2024-11-15T00:47:25,951 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2b9d412fef84da0a21312af9714e127, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731631641821 2024-11-15T00:47:25,951 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f58c66ad0f9748d98230adf2e9b0b018, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731631641175 2024-11-15T00:47:25,951 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c4694d7d4e54f1b9bf71acf2bcd7f3e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731631643132 2024-11-15T00:47:25,951 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a03c6fda8da743289ec5a60dec90adf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731631641821 2024-11-15T00:47:25,952 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 038af211ed5549c7b054063fc845975a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731631643132 2024-11-15T00:47:25,952 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14ae39b0e0cd41d09632e4c3c922a05d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731631643837 2024-11-15T00:47:25,952 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d1369f738ec441eb5e6bb527babad7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731631643837 2024-11-15T00:47:25,962 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:25,967 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#B#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:25,967 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/e02140d0661d4bcfbc0fbda2f6fedd54 is 50, key is test_row_0/B:col10/1731631643837/Put/seqid=0 2024-11-15T00:47:25,973 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e544478461004f52a743dc86141a70fd_8072ff2efb6653ab0fbbcdf205cf0e99 store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:25,974 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e544478461004f52a743dc86141a70fd_8072ff2efb6653ab0fbbcdf205cf0e99, store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:25,974 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e544478461004f52a743dc86141a70fd_8072ff2efb6653ab0fbbcdf205cf0e99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:25,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742462_1638 (size=12629) 2024-11-15T00:47:25,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742463_1639 (size=4469) 2024-11-15T00:47:25,988 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#A#compaction#536 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:25,988 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/203c87caace441e890dd00e5e40800ef is 175, key is test_row_0/A:col10/1731631643837/Put/seqid=0 2024-11-15T00:47:25,996 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/e02140d0661d4bcfbc0fbda2f6fedd54 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/e02140d0661d4bcfbc0fbda2f6fedd54 2024-11-15T00:47:25,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742464_1640 (size=31583) 2024-11-15T00:47:26,003 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/B of 8072ff2efb6653ab0fbbcdf205cf0e99 into e02140d0661d4bcfbc0fbda2f6fedd54(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:26,003 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:26,003 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/B, priority=12, startTime=1731631645947; duration=0sec 2024-11-15T00:47:26,003 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:26,003 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:B 2024-11-15T00:47:26,004 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:47:26,005 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:47:26,005 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/C is initiating minor compaction (all files) 2024-11-15T00:47:26,005 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/C in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:26,005 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/db3b2c4ebe1a4193adc7652f69835e56, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/bbc117530fc54a3e9d2e1c7cb0fb2916, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/9c1b1003c3664c09a1826a945d4e79a3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/f23def3431f145a4897f0207611ffc46] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=47.8 K 2024-11-15T00:47:26,005 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting db3b2c4ebe1a4193adc7652f69835e56, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731631641175 2024-11-15T00:47:26,005 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc117530fc54a3e9d2e1c7cb0fb2916, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731631641821 2024-11-15T00:47:26,006 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c1b1003c3664c09a1826a945d4e79a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731631643132 2024-11-15T00:47:26,006 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting f23def3431f145a4897f0207611ffc46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731631643837 2024-11-15T00:47:26,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T00:47:26,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:26,012 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:47:26,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:26,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:26,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:26,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:26,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:26,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:26,015 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#C#compaction#538 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:26,015 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/a75df293b1ed4ed09a4a556f21d1d61a is 50, key is test_row_0/C:col10/1731631643837/Put/seqid=0 2024-11-15T00:47:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T00:47:26,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742465_1641 (size=12629) 2024-11-15T00:47:26,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b5975856aa53417fbc576a1a533f252a_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631645110/Put/seqid=0 2024-11-15T00:47:26,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742466_1642 (size=12304) 2024-11-15T00:47:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:26,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:26,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631706275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631706276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631706280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T00:47:26,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631706391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631706392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631706396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,414 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/203c87caace441e890dd00e5e40800ef as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/203c87caace441e890dd00e5e40800ef 2024-11-15T00:47:26,431 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/A of 8072ff2efb6653ab0fbbcdf205cf0e99 into 203c87caace441e890dd00e5e40800ef(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:26,431 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:26,431 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/A, priority=12, startTime=1731631645947; duration=0sec 2024-11-15T00:47:26,431 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:26,431 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:A 2024-11-15T00:47:26,492 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/a75df293b1ed4ed09a4a556f21d1d61a as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/a75df293b1ed4ed09a4a556f21d1d61a 2024-11-15T00:47:26,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:26,522 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b5975856aa53417fbc576a1a533f252a_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b5975856aa53417fbc576a1a533f252a_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:26,525 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/C of 8072ff2efb6653ab0fbbcdf205cf0e99 into a75df293b1ed4ed09a4a556f21d1d61a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:26,525 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:26,525 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/C, priority=12, startTime=1731631645947; duration=0sec 2024-11-15T00:47:26,525 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:26,525 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:C 2024-11-15T00:47:26,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/f1343e1ecb6e413e8ce6949e3aeec60b, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:26,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/f1343e1ecb6e413e8ce6949e3aeec60b is 175, key is test_row_0/A:col10/1731631645110/Put/seqid=0 2024-11-15T00:47:26,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742467_1643 (size=31105) 2024-11-15T00:47:26,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631706600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631706609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631706612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T00:47:26,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631706920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631706932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631706932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:26,991 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/f1343e1ecb6e413e8ce6949e3aeec60b 2024-11-15T00:47:27,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/2b2de05407a64f56a11bacaa4c3d9032 is 50, key is test_row_0/B:col10/1731631645110/Put/seqid=0 2024-11-15T00:47:27,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742468_1644 (size=12151) 2024-11-15T00:47:27,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631707432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:27,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631707439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:27,444 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/2b2de05407a64f56a11bacaa4c3d9032 2024-11-15T00:47:27,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/6cbe44d686ac4ed5882bec6d725a840e is 50, key is test_row_0/C:col10/1731631645110/Put/seqid=0 2024-11-15T00:47:27,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631707449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:27,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:27,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742469_1645 (size=12151) 2024-11-15T00:47:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T00:47:27,855 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/6cbe44d686ac4ed5882bec6d725a840e 2024-11-15T00:47:27,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/f1343e1ecb6e413e8ce6949e3aeec60b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/f1343e1ecb6e413e8ce6949e3aeec60b 2024-11-15T00:47:27,864 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/f1343e1ecb6e413e8ce6949e3aeec60b, entries=150, sequenceid=245, filesize=30.4 K 2024-11-15T00:47:27,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/2b2de05407a64f56a11bacaa4c3d9032 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2b2de05407a64f56a11bacaa4c3d9032 2024-11-15T00:47:27,868 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2b2de05407a64f56a11bacaa4c3d9032, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T00:47:27,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/6cbe44d686ac4ed5882bec6d725a840e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/6cbe44d686ac4ed5882bec6d725a840e 2024-11-15T00:47:27,871 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/6cbe44d686ac4ed5882bec6d725a840e, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T00:47:27,872 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1860ms, sequenceid=245, compaction requested=false 2024-11-15T00:47:27,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:27,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:27,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-15T00:47:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-15T00:47:27,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-15T00:47:27,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-11-15T00:47:27,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.1770 sec 2024-11-15T00:47:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:28,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T00:47:28,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:28,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:28,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:28,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:28,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:28,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:28,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159f151065ce6b43e386c510b8e192d533_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631648455/Put/seqid=0 2024-11-15T00:47:28,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742470_1646 (size=12354) 2024-11-15T00:47:28,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631708482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:28,488 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:28,491 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159f151065ce6b43e386c510b8e192d533_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159f151065ce6b43e386c510b8e192d533_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:28,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:28,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631708485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631708483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:28,492 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/36a9a7f0e9874d8ab8a8c6259fd5cad4, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:28,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/36a9a7f0e9874d8ab8a8c6259fd5cad4 is 175, key is test_row_0/A:col10/1731631648455/Put/seqid=0 2024-11-15T00:47:28,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742471_1647 (size=31155) 2024-11-15T00:47:28,514 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/36a9a7f0e9874d8ab8a8c6259fd5cad4 2024-11-15T00:47:28,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/d324ed78f37044fcb609dd3da0162846 is 50, key is test_row_0/B:col10/1731631648455/Put/seqid=0 2024-11-15T00:47:28,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742472_1648 (size=12201) 2024-11-15T00:47:28,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631708590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:28,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631708599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631708600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:28,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631708799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:28,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631708809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:28,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631708811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:28,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/d324ed78f37044fcb609dd3da0162846 2024-11-15T00:47:28,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/e907ba3c85894f8898812817bc75d2e1 is 50, key is test_row_0/C:col10/1731631648455/Put/seqid=0 2024-11-15T00:47:28,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742473_1649 (size=12201) 2024-11-15T00:47:29,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/e907ba3c85894f8898812817bc75d2e1 2024-11-15T00:47:29,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/36a9a7f0e9874d8ab8a8c6259fd5cad4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/36a9a7f0e9874d8ab8a8c6259fd5cad4 2024-11-15T00:47:29,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/36a9a7f0e9874d8ab8a8c6259fd5cad4, entries=150, sequenceid=261, filesize=30.4 K 2024-11-15T00:47:29,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/d324ed78f37044fcb609dd3da0162846 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/d324ed78f37044fcb609dd3da0162846 2024-11-15T00:47:29,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/d324ed78f37044fcb609dd3da0162846, entries=150, sequenceid=261, filesize=11.9 K 2024-11-15T00:47:29,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/e907ba3c85894f8898812817bc75d2e1 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/e907ba3c85894f8898812817bc75d2e1 2024-11-15T00:47:29,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631709112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631709119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631709119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/e907ba3c85894f8898812817bc75d2e1, entries=150, sequenceid=261, filesize=11.9 K 2024-11-15T00:47:29,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 669ms, sequenceid=261, compaction requested=true 2024-11-15T00:47:29,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:29,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:29,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:29,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:47:29,126 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:29,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:29,126 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:29,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:47:29,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:29,134 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:29,134 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/B is initiating minor compaction (all files) 2024-11-15T00:47:29,134 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/B in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:29,134 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/e02140d0661d4bcfbc0fbda2f6fedd54, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2b2de05407a64f56a11bacaa4c3d9032, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/d324ed78f37044fcb609dd3da0162846] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=36.1 K 2024-11-15T00:47:29,135 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e02140d0661d4bcfbc0fbda2f6fedd54, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731631643837 2024-11-15T00:47:29,135 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2de05407a64f56a11bacaa4c3d9032, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631645100 2024-11-15T00:47:29,135 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:29,135 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/A is initiating minor compaction (all files) 2024-11-15T00:47:29,135 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/A in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:29,135 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/203c87caace441e890dd00e5e40800ef, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/f1343e1ecb6e413e8ce6949e3aeec60b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/36a9a7f0e9874d8ab8a8c6259fd5cad4] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=91.6 K 2024-11-15T00:47:29,136 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:29,136 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. files: [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/203c87caace441e890dd00e5e40800ef, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/f1343e1ecb6e413e8ce6949e3aeec60b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/36a9a7f0e9874d8ab8a8c6259fd5cad4] 2024-11-15T00:47:29,140 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting d324ed78f37044fcb609dd3da0162846, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731631646267 2024-11-15T00:47:29,140 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203c87caace441e890dd00e5e40800ef, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731631643837 2024-11-15T00:47:29,141 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1343e1ecb6e413e8ce6949e3aeec60b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631645100 2024-11-15T00:47:29,142 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36a9a7f0e9874d8ab8a8c6259fd5cad4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731631646267 2024-11-15T00:47:29,152 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:29,167 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#B#compaction#546 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:29,167 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/f7f64a4d681e4e13bee8a79c57e172cd is 50, key is test_row_0/B:col10/1731631648455/Put/seqid=0 2024-11-15T00:47:29,185 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115071801b622de48f485cb38fd612d496d_8072ff2efb6653ab0fbbcdf205cf0e99 store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:29,187 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115071801b622de48f485cb38fd612d496d_8072ff2efb6653ab0fbbcdf205cf0e99, store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:29,187 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115071801b622de48f485cb38fd612d496d_8072ff2efb6653ab0fbbcdf205cf0e99 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:29,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742474_1650 (size=12781) 2024-11-15T00:47:29,220 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/f7f64a4d681e4e13bee8a79c57e172cd as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f7f64a4d681e4e13bee8a79c57e172cd 2024-11-15T00:47:29,224 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/B of 8072ff2efb6653ab0fbbcdf205cf0e99 into f7f64a4d681e4e13bee8a79c57e172cd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:29,225 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:29,225 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/B, priority=13, startTime=1731631649126; duration=0sec 2024-11-15T00:47:29,225 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:47:29,225 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:B 2024-11-15T00:47:29,225 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:47:29,227 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:47:29,227 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1541): 8072ff2efb6653ab0fbbcdf205cf0e99/C is initiating minor compaction (all files) 2024-11-15T00:47:29,227 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8072ff2efb6653ab0fbbcdf205cf0e99/C in TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:29,227 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/a75df293b1ed4ed09a4a556f21d1d61a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/6cbe44d686ac4ed5882bec6d725a840e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/e907ba3c85894f8898812817bc75d2e1] into tmpdir=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp, totalSize=36.1 K 2024-11-15T00:47:29,227 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting a75df293b1ed4ed09a4a556f21d1d61a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731631643837 2024-11-15T00:47:29,228 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cbe44d686ac4ed5882bec6d725a840e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631645100 2024-11-15T00:47:29,228 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] compactions.Compactor(225): Compacting e907ba3c85894f8898812817bc75d2e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731631646267 2024-11-15T00:47:29,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742475_1651 (size=4469) 2024-11-15T00:47:29,235 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#A#compaction#545 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:29,236 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/fe5d84302287425e85ed4bbd4a2e3fcb is 175, key is test_row_0/A:col10/1731631648455/Put/seqid=0 2024-11-15T00:47:29,243 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8072ff2efb6653ab0fbbcdf205cf0e99#C#compaction#547 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:47:29,243 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/7b39f0b0851f43929944083125930827 is 50, key is test_row_0/C:col10/1731631648455/Put/seqid=0 2024-11-15T00:47:29,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742476_1652 (size=31735) 2024-11-15T00:47:29,255 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/fe5d84302287425e85ed4bbd4a2e3fcb as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/fe5d84302287425e85ed4bbd4a2e3fcb 2024-11-15T00:47:29,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742477_1653 (size=12781) 2024-11-15T00:47:29,259 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/A of 8072ff2efb6653ab0fbbcdf205cf0e99 into fe5d84302287425e85ed4bbd4a2e3fcb(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:29,260 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:29,260 INFO [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/A, priority=13, startTime=1731631649126; duration=0sec 2024-11-15T00:47:29,260 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:29,260 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:A 2024-11-15T00:47:29,262 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/7b39f0b0851f43929944083125930827 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/7b39f0b0851f43929944083125930827 2024-11-15T00:47:29,268 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8072ff2efb6653ab0fbbcdf205cf0e99/C of 8072ff2efb6653ab0fbbcdf205cf0e99 into 7b39f0b0851f43929944083125930827(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:47:29,268 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:29,268 INFO [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., storeName=8072ff2efb6653ab0fbbcdf205cf0e99/C, priority=13, startTime=1731631649126; duration=0sec 2024-11-15T00:47:29,268 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:29,268 DEBUG [RS:0;fb2f6cb7846a:34537-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:C 2024-11-15T00:47:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:29,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T00:47:29,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:29,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:29,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:29,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:29,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:29,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:29,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150fcbdea95cce43b6b606e42599dfab62_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631649631/Put/seqid=0 2024-11-15T00:47:29,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742478_1654 (size=12454) 2024-11-15T00:47:29,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631709643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631709647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631709647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631709749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631709759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631709759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T00:47:29,870 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:29,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:47:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-15T00:47:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T00:47:29,872 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:47:29,873 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:47:29,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:47:29,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631709959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631709970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631709970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T00:47:30,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T00:47:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,044 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:30,047 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150fcbdea95cce43b6b606e42599dfab62_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150fcbdea95cce43b6b606e42599dfab62_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:30,048 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/cf16bf69dc164fceaf8c34de1f3d0956, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:30,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/cf16bf69dc164fceaf8c34de1f3d0956 is 175, key is test_row_0/A:col10/1731631649631/Put/seqid=0 2024-11-15T00:47:30,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742479_1655 (size=31255) 2024-11-15T00:47:30,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T00:47:30,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:30,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T00:47:30,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631710269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:30,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631710280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631710280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:30,334 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:47:30,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T00:47:30,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:30,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,460 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/cf16bf69dc164fceaf8c34de1f3d0956 2024-11-15T00:47:30,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/86e05e65db314f84842081ea4d0b0dbf is 50, key is test_row_0/B:col10/1731631649631/Put/seqid=0 2024-11-15T00:47:30,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T00:47:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T00:47:30,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742480_1656 (size=12301) 2024-11-15T00:47:30,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/86e05e65db314f84842081ea4d0b0dbf 2024-11-15T00:47:30,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/12f6704da9de462c935e4f207b4bd100 is 50, key is test_row_0/C:col10/1731631649631/Put/seqid=0 2024-11-15T00:47:30,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742481_1657 (size=12301) 2024-11-15T00:47:30,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T00:47:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631710779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:30,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631710790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631710790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:30,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T00:47:30,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:30,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T00:47:30,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:30,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:47:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T00:47:31,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/12f6704da9de462c935e4f207b4bd100 2024-11-15T00:47:31,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/cf16bf69dc164fceaf8c34de1f3d0956 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/cf16bf69dc164fceaf8c34de1f3d0956 2024-11-15T00:47:31,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/cf16bf69dc164fceaf8c34de1f3d0956, entries=150, sequenceid=284, filesize=30.5 K 2024-11-15T00:47:31,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/86e05e65db314f84842081ea4d0b0dbf as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/86e05e65db314f84842081ea4d0b0dbf 2024-11-15T00:47:31,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/86e05e65db314f84842081ea4d0b0dbf, entries=150, sequenceid=284, filesize=12.0 K 2024-11-15T00:47:31,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/12f6704da9de462c935e4f207b4bd100 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/12f6704da9de462c935e4f207b4bd100 2024-11-15T00:47:31,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/12f6704da9de462c935e4f207b4bd100, entries=150, sequenceid=284, filesize=12.0 K 2024-11-15T00:47:31,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1398ms, sequenceid=284, compaction requested=false 2024-11-15T00:47:31,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:31,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34537 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T00:47:31,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:31,114 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T00:47:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T00:47:31,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150bfa8e226daf4e08918dfa3f43d95bef_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631649645/Put/seqid=0 2024-11-15T00:47:31,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742482_1658 (size=12454) 2024-11-15T00:47:31,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:31,174 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150bfa8e226daf4e08918dfa3f43d95bef_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150bfa8e226daf4e08918dfa3f43d95bef_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:31,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/95b3b33f6b9a4575918dda37e473515f, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:31,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/95b3b33f6b9a4575918dda37e473515f is 175, key is test_row_0/A:col10/1731631649645/Put/seqid=0 2024-11-15T00:47:31,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742483_1659 (size=31255) 2024-11-15T00:47:31,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. as already flushing 2024-11-15T00:47:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631711435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:31,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631711439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:31,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631711549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631711549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:31,612 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/95b3b33f6b9a4575918dda37e473515f 2024-11-15T00:47:31,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/f37d59623ca142e29ab4b93e32ccb761 is 50, key is test_row_0/B:col10/1731631649645/Put/seqid=0 2024-11-15T00:47:31,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742484_1660 (size=12301) 2024-11-15T00:47:31,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631711759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631711760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:31,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44944 deadline: 1731631711799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:31,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44928 deadline: 1731631711799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:31,804 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_2' in 8072ff2efb6653ab0fbbcdf205cf0e99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:47:31,804 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in 8072ff2efb6653ab0fbbcdf205cf0e99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:47:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:31,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44870 deadline: 1731631711809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:31,812 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,34537,1731631472208 for 'test_row_0' in 8072ff2efb6653ab0fbbcdf205cf0e99 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:47:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T00:47:32,035 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/f37d59623ca142e29ab4b93e32ccb761 2024-11-15T00:47:32,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/4f95e6b09cd945d3bb464e1e0f95f703 is 50, key is test_row_0/C:col10/1731631649645/Put/seqid=0 2024-11-15T00:47:32,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742485_1661 (size=12301) 2024-11-15T00:47:32,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44898 deadline: 1731631712069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:32,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:32,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:47:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34537 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44856 deadline: 1731631712072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:47:32,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., hostname=fb2f6cb7846a,34537,1731631472208, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:47:32,452 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/4f95e6b09cd945d3bb464e1e0f95f703 2024-11-15T00:47:32,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/95b3b33f6b9a4575918dda37e473515f as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/95b3b33f6b9a4575918dda37e473515f 2024-11-15T00:47:32,504 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/95b3b33f6b9a4575918dda37e473515f, entries=150, sequenceid=296, filesize=30.5 K 2024-11-15T00:47:32,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/f37d59623ca142e29ab4b93e32ccb761 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f37d59623ca142e29ab4b93e32ccb761 2024-11-15T00:47:32,507 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f37d59623ca142e29ab4b93e32ccb761, entries=150, sequenceid=296, filesize=12.0 K 2024-11-15T00:47:32,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/4f95e6b09cd945d3bb464e1e0f95f703 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4f95e6b09cd945d3bb464e1e0f95f703 2024-11-15T00:47:32,511 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4f95e6b09cd945d3bb464e1e0f95f703, entries=150, sequenceid=296, filesize=12.0 K 2024-11-15T00:47:32,512 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1398ms, sequenceid=296, compaction requested=true 2024-11-15T00:47:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-15T00:47:32,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-15T00:47:32,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-15T00:47:32,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6410 sec 2024-11-15T00:47:32,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.6450 sec 2024-11-15T00:47:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34537 {}] regionserver.HRegion(8855): Flush requested on 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:32,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T00:47:32,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631651414, max timestamp=1731631652581Num uniques 859; 2024-11-15T00:47:32,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:32,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631651414, max timestamp=1731631652581Num uniques 859; 2024-11-15T00:47:32,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:32,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631651414, max timestamp=1731631652581Num uniques 859; 2024-11-15T00:47:32,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:32,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:32,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:32,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:32,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:32,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:32,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:32,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111563dfe90759e54415b2af23ebd20cf3ae_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631652581/Put/seqid=0 2024-11-15T00:47:32,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742486_1662 (size=12454) 2024-11-15T00:47:32,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631652620, max timestamp=1731631652641Num uniques 859; 2024-11-15T00:47:32,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:32,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631652620, max timestamp=1731631652641Num uniques 859; 2024-11-15T00:47:32,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:32,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731631652620, max timestamp=1731631652641Num uniques 859; 2024-11-15T00:47:32,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T00:47:32,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731631652674, max timestamp=1731631652693Num uniques 1018; 2024-11-15T00:47:32,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-15T00:47:32,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731631652674, max timestamp=1731631652693Num uniques 1018; 2024-11-15T00:47:32,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731631652674, max timestamp=1731631652693Num uniques 1018; 2024-11-15T00:47:32,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-15T00:47:32,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-15T00:47:32,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731631652723, max timestamp=1731631652740Num uniques 1077; 2024-11-15T00:47:32,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-15T00:47:32,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731631652723, max timestamp=1731631652740Num uniques 1077; 2024-11-15T00:47:32,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731631652723, max timestamp=1731631652740Num uniques 1077; 2024-11-15T00:47:32,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-15T00:47:32,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-15T00:47:32,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731631652799, max timestamp=1731631652809Num uniques 1086; 2024-11-15T00:47:32,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-15T00:47:32,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731631652799, max timestamp=1731631652809Num uniques 1086; 2024-11-15T00:47:32,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-15T00:47:32,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731631652799, max timestamp=1731631652809Num uniques 1086; 2024-11-15T00:47:32,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-15T00:47:32,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731631652848, max timestamp=1731631652861Num uniques 1045; 2024-11-15T00:47:32,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731631652848, max timestamp=1731631652861Num uniques 1045; 2024-11-15T00:47:32,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731631652848, max timestamp=1731631652861Num uniques 1045; 2024-11-15T00:47:32,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-15T00:47:32,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-15T00:47:32,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-15T00:47:32,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731631652902, max timestamp=1731631652922Num uniques 1104; 2024-11-15T00:47:32,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731631652902, max timestamp=1731631652922Num uniques 1104; 2024-11-15T00:47:32,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-15T00:47:32,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-15T00:47:32,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731631652902, max timestamp=1731631652922Num uniques 1104; 2024-11-15T00:47:32,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-15T00:47:32,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731631652968, max timestamp=1731631652978Num uniques 1013; 2024-11-15T00:47:32,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-15T00:47:32,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731631652968, max timestamp=1731631652978Num uniques 1013; 2024-11-15T00:47:32,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731631652968, max timestamp=1731631652978Num uniques 1013; 2024-11-15T00:47:32,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-15T00:47:32,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-15T00:47:33,006 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:33,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111563dfe90759e54415b2af23ebd20cf3ae_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111563dfe90759e54415b2af23ebd20cf3ae_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:33,009 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/60f7abb36bb54bb7a351e1ce862535b3, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:33,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/60f7abb36bb54bb7a351e1ce862535b3 is 175, key is test_row_0/A:col10/1731631652581/Put/seqid=0 2024-11-15T00:47:33,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742487_1663 (size=31255) 2024-11-15T00:47:33,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731631653025, max timestamp=1731631653039Num uniques 1022; 2024-11-15T00:47:33,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-15T00:47:33,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731631653025, max timestamp=1731631653039Num uniques 1022; 2024-11-15T00:47:33,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-15T00:47:33,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731631653025, max timestamp=1731631653039Num uniques 1022; 2024-11-15T00:47:33,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fb2f6cb7846a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-15T00:47:33,052 INFO [Thread-2592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2592. 2024-11-15T00:47:33,052 INFO [Thread-2588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2588. 2024-11-15T00:47:33,052 DEBUG [Thread-2592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:33,052 INFO [Thread-2577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2577. 2024-11-15T00:47:33,052 DEBUG [Thread-2588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:33,052 DEBUG [Thread-2592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,052 DEBUG [Thread-2588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,052 DEBUG [Thread-2577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:33,052 DEBUG [Thread-2577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,052 DEBUG [Thread-2592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,052 DEBUG [Thread-2577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:33,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:33,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:33,053 DEBUG [Thread-2588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,056 INFO [Thread-2590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2590. 2024-11-15T00:47:33,056 DEBUG [Thread-2590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:33,056 DEBUG [Thread-2590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,056 DEBUG [Thread-2590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:33,056 INFO [Thread-2581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2581. 2024-11-15T00:47:33,056 DEBUG [Thread-2581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:33,056 DEBUG [Thread-2581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,056 DEBUG [Thread-2581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:33,057 INFO [Thread-2594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2594. 2024-11-15T00:47:33,057 DEBUG [Thread-2594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:33,057 DEBUG [Thread-2594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,057 DEBUG [Thread-2594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,057 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:33,058 INFO [Thread-2596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2596. 2024-11-15T00:47:33,058 DEBUG [Thread-2596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:33,058 DEBUG [Thread-2596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,058 DEBUG [Thread-2596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:33,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/60f7abb36bb54bb7a351e1ce862535b3 2024-11-15T00:47:33,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/9fb0d457d3444f058f04c71f6525688e is 50, key is test_row_0/B:col10/1731631652581/Put/seqid=0 2024-11-15T00:47:33,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742488_1664 (size=12301) 2024-11-15T00:47:33,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/9fb0d457d3444f058f04c71f6525688e 2024-11-15T00:47:33,823 INFO [Thread-2583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2583. 2024-11-15T00:47:33,823 INFO [Thread-2579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2579. 2024-11-15T00:47:33,823 DEBUG [Thread-2579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:33,823 DEBUG [Thread-2583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:33,823 DEBUG [Thread-2583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,823 DEBUG [Thread-2579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,823 DEBUG [Thread-2583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,823 DEBUG [Thread-2579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:33,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:33,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/4ea6a70cff054b0fae5899bca2143f22 is 50, key is test_row_0/C:col10/1731631652581/Put/seqid=0 2024-11-15T00:47:33,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742489_1665 (size=12301) 2024-11-15T00:47:33,840 INFO [Thread-2585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2585. 2024-11-15T00:47:33,840 DEBUG [Thread-2585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:47:33,840 DEBUG [Thread-2585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,840 DEBUG [Thread-2585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:33,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T00:47:34,029 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2525 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2596 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2365 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2521 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2579 2024-11-15T00:47:34,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:47:34,030 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:47:34,030 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:47:34,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:34,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:34,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:34,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:47:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T00:47:34,033 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631654033"}]},"ts":"1731631654033"} 2024-11-15T00:47:34,034 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:47:34,034 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:47:34,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:47:34,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, UNASSIGN}] 2024-11-15T00:47:34,036 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, UNASSIGN 2024-11-15T00:47:34,036 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=8072ff2efb6653ab0fbbcdf205cf0e99, regionState=CLOSING, regionLocation=fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:34,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, UNASSIGN because future has completed 2024-11-15T00:47:34,038 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:47:34,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208}] 2024-11-15T00:47:34,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T00:47:34,190 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(122): Close 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:34,190 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:47:34,190 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1722): Closing 8072ff2efb6653ab0fbbcdf205cf0e99, disabling compactions & flushes 2024-11-15T00:47:34,190 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:34,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/4ea6a70cff054b0fae5899bca2143f22 2024-11-15T00:47:34,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/60f7abb36bb54bb7a351e1ce862535b3 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/60f7abb36bb54bb7a351e1ce862535b3 2024-11-15T00:47:34,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/60f7abb36bb54bb7a351e1ce862535b3, entries=150, sequenceid=318, filesize=30.5 K 2024-11-15T00:47:34,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/9fb0d457d3444f058f04c71f6525688e as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/9fb0d457d3444f058f04c71f6525688e 2024-11-15T00:47:34,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/9fb0d457d3444f058f04c71f6525688e, entries=150, sequenceid=318, filesize=12.0 K 2024-11-15T00:47:34,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/4ea6a70cff054b0fae5899bca2143f22 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4ea6a70cff054b0fae5899bca2143f22 2024-11-15T00:47:34,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4ea6a70cff054b0fae5899bca2143f22, entries=150, sequenceid=318, filesize=12.0 K 2024-11-15T00:47:34,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=87.22 KB/89310 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1669ms, sequenceid=318, compaction requested=true 2024-11-15T00:47:34,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8072ff2efb6653ab0fbbcdf205cf0e99: 2024-11-15T00:47:34,252 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:34,252 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:34,252 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. after waiting 0 ms 2024-11-15T00:47:34,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:34,252 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. because compaction request was cancelled 2024-11-15T00:47:34,252 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:34,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:34,252 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:A 2024-11-15T00:47:34,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:B, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:34,252 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. because compaction request was cancelled 2024-11-15T00:47:34,252 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(2902): Flushing 8072ff2efb6653ab0fbbcdf205cf0e99 3/3 column families, dataSize=87.22 KB heapSize=239.63 KB 2024-11-15T00:47:34,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:34,252 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:B 2024-11-15T00:47:34,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8072ff2efb6653ab0fbbcdf205cf0e99:C, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:47:34,252 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. because compaction request was cancelled 2024-11-15T00:47:34,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:47:34,252 DEBUG [RS:0;fb2f6cb7846a:34537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8072ff2efb6653ab0fbbcdf205cf0e99:C 2024-11-15T00:47:34,252 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=A 2024-11-15T00:47:34,252 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:34,252 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=B 2024-11-15T00:47:34,252 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:34,252 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8072ff2efb6653ab0fbbcdf205cf0e99, store=C 2024-11-15T00:47:34,252 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T00:47:34,259 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159e8237fe82df44b9bedf2f45c1ded2a3_8072ff2efb6653ab0fbbcdf205cf0e99 is 50, key is test_row_0/A:col10/1731631653839/Put/seqid=0 2024-11-15T00:47:34,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742490_1666 (size=12454) 2024-11-15T00:47:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T00:47:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T00:47:34,666 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:47:34,669 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159e8237fe82df44b9bedf2f45c1ded2a3_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e8237fe82df44b9bedf2f45c1ded2a3_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:34,670 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/767a359b7fba4486af4ecd78b79ff4b4, store: [table=TestAcidGuarantees family=A region=8072ff2efb6653ab0fbbcdf205cf0e99] 2024-11-15T00:47:34,671 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/767a359b7fba4486af4ecd78b79ff4b4 is 175, key is test_row_0/A:col10/1731631653839/Put/seqid=0 2024-11-15T00:47:34,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742491_1667 (size=31255) 2024-11-15T00:47:34,680 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=467, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/767a359b7fba4486af4ecd78b79ff4b4 2024-11-15T00:47:34,687 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/410697313ece4c0380dc831191ac6faf is 50, key is test_row_0/B:col10/1731631653839/Put/seqid=0 2024-11-15T00:47:34,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742492_1668 (size=12301) 2024-11-15T00:47:35,102 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/410697313ece4c0380dc831191ac6faf 2024-11-15T00:47:35,108 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/fa5bc57d87e04fb299646a8e79ed3e35 is 50, key is test_row_0/C:col10/1731631653839/Put/seqid=0 2024-11-15T00:47:35,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742493_1669 (size=12301) 2024-11-15T00:47:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T00:47:35,513 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/fa5bc57d87e04fb299646a8e79ed3e35 2024-11-15T00:47:35,517 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/A/767a359b7fba4486af4ecd78b79ff4b4 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/767a359b7fba4486af4ecd78b79ff4b4 2024-11-15T00:47:35,520 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/767a359b7fba4486af4ecd78b79ff4b4, entries=150, sequenceid=467, filesize=30.5 K 2024-11-15T00:47:35,521 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/B/410697313ece4c0380dc831191ac6faf as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/410697313ece4c0380dc831191ac6faf 2024-11-15T00:47:35,525 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/410697313ece4c0380dc831191ac6faf, entries=150, sequenceid=467, filesize=12.0 K 2024-11-15T00:47:35,525 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/.tmp/C/fa5bc57d87e04fb299646a8e79ed3e35 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/fa5bc57d87e04fb299646a8e79ed3e35 2024-11-15T00:47:35,529 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/fa5bc57d87e04fb299646a8e79ed3e35, entries=150, sequenceid=467, filesize=12.0 K 2024-11-15T00:47:35,530 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~239.58 KB/245328, currentSize=0 B/0 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1277ms, sequenceid=467, compaction requested=true 2024-11-15T00:47:35,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/22638295d9d14264941a09fb892a63cb, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0d7cdac77b914e529903734c8e15316b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/2e082444302645c797fe05148e744b29, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/d137db0de05e485fb89c6d7f4920c3e4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/5b856442525a4fba9ecd9b9cc07da686, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0897c85882844c2a9557d9cf49ec8364, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/409e3a6ab55b4f4ea00509b60d03452f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/89e08a01ad23447b99608e66e4b2f55d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/7f6eceae915144eea474102a7edea122, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b7d3cbc765584a0888f01a80113367c6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2d3bce07cc543b9b28e25d7df925ea9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2b9d412fef84da0a21312af9714e127, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/14ae39b0e0cd41d09632e4c3c922a05d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/203c87caace441e890dd00e5e40800ef, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/f1343e1ecb6e413e8ce6949e3aeec60b, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/36a9a7f0e9874d8ab8a8c6259fd5cad4] to archive 2024-11-15T00:47:35,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:47:35,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/22638295d9d14264941a09fb892a63cb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/22638295d9d14264941a09fb892a63cb 2024-11-15T00:47:35,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0d7cdac77b914e529903734c8e15316b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0d7cdac77b914e529903734c8e15316b 2024-11-15T00:47:35,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/2e082444302645c797fe05148e744b29 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/2e082444302645c797fe05148e744b29 2024-11-15T00:47:35,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/d137db0de05e485fb89c6d7f4920c3e4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/d137db0de05e485fb89c6d7f4920c3e4 2024-11-15T00:47:35,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/5b856442525a4fba9ecd9b9cc07da686 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/5b856442525a4fba9ecd9b9cc07da686 2024-11-15T00:47:35,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0897c85882844c2a9557d9cf49ec8364 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/0897c85882844c2a9557d9cf49ec8364 2024-11-15T00:47:35,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/409e3a6ab55b4f4ea00509b60d03452f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/409e3a6ab55b4f4ea00509b60d03452f 2024-11-15T00:47:35,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/89e08a01ad23447b99608e66e4b2f55d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/89e08a01ad23447b99608e66e4b2f55d 2024-11-15T00:47:35,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/7f6eceae915144eea474102a7edea122 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/7f6eceae915144eea474102a7edea122 2024-11-15T00:47:35,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b7d3cbc765584a0888f01a80113367c6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b7d3cbc765584a0888f01a80113367c6 2024-11-15T00:47:35,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2d3bce07cc543b9b28e25d7df925ea9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2d3bce07cc543b9b28e25d7df925ea9 2024-11-15T00:47:35,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2b9d412fef84da0a21312af9714e127 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/b2b9d412fef84da0a21312af9714e127 2024-11-15T00:47:35,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/3c4694d7d4e54f1b9bf71acf2bcd7f3e 2024-11-15T00:47:35,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/14ae39b0e0cd41d09632e4c3c922a05d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/14ae39b0e0cd41d09632e4c3c922a05d 2024-11-15T00:47:35,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/203c87caace441e890dd00e5e40800ef to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/203c87caace441e890dd00e5e40800ef 2024-11-15T00:47:35,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/f1343e1ecb6e413e8ce6949e3aeec60b to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/f1343e1ecb6e413e8ce6949e3aeec60b 2024-11-15T00:47:35,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/36a9a7f0e9874d8ab8a8c6259fd5cad4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/36a9a7f0e9874d8ab8a8c6259fd5cad4 2024-11-15T00:47:35,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/3a499b6902894fffbe00dad0998fe683, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/c8799593c5e4436e95140eb627c67e72, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/70a5dddceddb4d949b4bd5b39630ef27, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f2f96ac7ae344160ab9267ef35ea9664, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8673079268b44555a8a2395e1e96eb32, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/86be62d9de084e1cb561ef89c4e725fc, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8c804ffb9dc043059d762d75713d6007, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/47548f2e5b8f42cf9decf6391ada48ea, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/1fda559ce2eb4930a4897f8d29ac1a80, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f58c66ad0f9748d98230adf2e9b0b018, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/069d68476b374454b22e8f526d007a5a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/a03c6fda8da743289ec5a60dec90adf5, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/038af211ed5549c7b054063fc845975a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/e02140d0661d4bcfbc0fbda2f6fedd54, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2d1369f738ec441eb5e6bb527babad7f, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2b2de05407a64f56a11bacaa4c3d9032, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/d324ed78f37044fcb609dd3da0162846] to archive 2024-11-15T00:47:35,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:47:35,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/3a499b6902894fffbe00dad0998fe683 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/3a499b6902894fffbe00dad0998fe683 2024-11-15T00:47:35,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/c8799593c5e4436e95140eb627c67e72 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/c8799593c5e4436e95140eb627c67e72 2024-11-15T00:47:35,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/70a5dddceddb4d949b4bd5b39630ef27 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/70a5dddceddb4d949b4bd5b39630ef27 2024-11-15T00:47:35,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f2f96ac7ae344160ab9267ef35ea9664 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f2f96ac7ae344160ab9267ef35ea9664 2024-11-15T00:47:35,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8673079268b44555a8a2395e1e96eb32 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8673079268b44555a8a2395e1e96eb32 2024-11-15T00:47:35,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/86be62d9de084e1cb561ef89c4e725fc to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/86be62d9de084e1cb561ef89c4e725fc 2024-11-15T00:47:35,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8c804ffb9dc043059d762d75713d6007 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/8c804ffb9dc043059d762d75713d6007 2024-11-15T00:47:35,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/47548f2e5b8f42cf9decf6391ada48ea to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/47548f2e5b8f42cf9decf6391ada48ea 2024-11-15T00:47:35,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/1fda559ce2eb4930a4897f8d29ac1a80 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/1fda559ce2eb4930a4897f8d29ac1a80 2024-11-15T00:47:35,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f58c66ad0f9748d98230adf2e9b0b018 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f58c66ad0f9748d98230adf2e9b0b018 2024-11-15T00:47:35,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/069d68476b374454b22e8f526d007a5a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/069d68476b374454b22e8f526d007a5a 2024-11-15T00:47:35,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/a03c6fda8da743289ec5a60dec90adf5 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/a03c6fda8da743289ec5a60dec90adf5 2024-11-15T00:47:35,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/038af211ed5549c7b054063fc845975a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/038af211ed5549c7b054063fc845975a 2024-11-15T00:47:35,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/e02140d0661d4bcfbc0fbda2f6fedd54 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/e02140d0661d4bcfbc0fbda2f6fedd54 2024-11-15T00:47:35,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2d1369f738ec441eb5e6bb527babad7f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2d1369f738ec441eb5e6bb527babad7f 2024-11-15T00:47:35,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2b2de05407a64f56a11bacaa4c3d9032 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/2b2de05407a64f56a11bacaa4c3d9032 2024-11-15T00:47:35,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/d324ed78f37044fcb609dd3da0162846 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/d324ed78f37044fcb609dd3da0162846 2024-11-15T00:47:35,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/01e40590fd854a6da7ae9d410ce5a2c6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/fdca5f98b3ee40aa90590d4998803df4, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/659ccc59a8d2494c8ba5a0e0a5b4de3a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/5423db67d93547a290ea8ba9fcf73c8e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ef59e66eb7284dae86cc7828276f6df6, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/7e2e1f883f984bff982d5cda21c1ba2d, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ebdf68e7f4b4424999604a1dd7b138bd, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4459a81b1b76428c9d0f2e706f51f0d9, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ba9f83175a884a788757592ff37a286c, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/db3b2c4ebe1a4193adc7652f69835e56, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/099574a236ee41ef98eaa7c1dad77ef1, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/bbc117530fc54a3e9d2e1c7cb0fb2916, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/9c1b1003c3664c09a1826a945d4e79a3, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/a75df293b1ed4ed09a4a556f21d1d61a, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/f23def3431f145a4897f0207611ffc46, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/6cbe44d686ac4ed5882bec6d725a840e, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/e907ba3c85894f8898812817bc75d2e1] to archive 2024-11-15T00:47:35,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:47:35,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/01e40590fd854a6da7ae9d410ce5a2c6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/01e40590fd854a6da7ae9d410ce5a2c6 2024-11-15T00:47:35,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/fdca5f98b3ee40aa90590d4998803df4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/fdca5f98b3ee40aa90590d4998803df4 2024-11-15T00:47:35,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/659ccc59a8d2494c8ba5a0e0a5b4de3a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/659ccc59a8d2494c8ba5a0e0a5b4de3a 2024-11-15T00:47:35,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/5423db67d93547a290ea8ba9fcf73c8e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/5423db67d93547a290ea8ba9fcf73c8e 2024-11-15T00:47:35,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ef59e66eb7284dae86cc7828276f6df6 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ef59e66eb7284dae86cc7828276f6df6 2024-11-15T00:47:35,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/7e2e1f883f984bff982d5cda21c1ba2d to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/7e2e1f883f984bff982d5cda21c1ba2d 2024-11-15T00:47:35,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ebdf68e7f4b4424999604a1dd7b138bd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ebdf68e7f4b4424999604a1dd7b138bd 2024-11-15T00:47:35,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4459a81b1b76428c9d0f2e706f51f0d9 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4459a81b1b76428c9d0f2e706f51f0d9 2024-11-15T00:47:35,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ba9f83175a884a788757592ff37a286c to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/ba9f83175a884a788757592ff37a286c 2024-11-15T00:47:35,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/db3b2c4ebe1a4193adc7652f69835e56 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/db3b2c4ebe1a4193adc7652f69835e56 2024-11-15T00:47:35,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/099574a236ee41ef98eaa7c1dad77ef1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/099574a236ee41ef98eaa7c1dad77ef1 2024-11-15T00:47:35,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/bbc117530fc54a3e9d2e1c7cb0fb2916 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/bbc117530fc54a3e9d2e1c7cb0fb2916 2024-11-15T00:47:35,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/9c1b1003c3664c09a1826a945d4e79a3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/9c1b1003c3664c09a1826a945d4e79a3 2024-11-15T00:47:35,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/a75df293b1ed4ed09a4a556f21d1d61a to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/a75df293b1ed4ed09a4a556f21d1d61a 2024-11-15T00:47:35,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/f23def3431f145a4897f0207611ffc46 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/f23def3431f145a4897f0207611ffc46 2024-11-15T00:47:35,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/6cbe44d686ac4ed5882bec6d725a840e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/6cbe44d686ac4ed5882bec6d725a840e 2024-11-15T00:47:35,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/e907ba3c85894f8898812817bc75d2e1 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/e907ba3c85894f8898812817bc75d2e1 2024-11-15T00:47:35,585 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/recovered.edits/470.seqid, newMaxSeqId=470, maxSeqId=4 2024-11-15T00:47:35,585 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. 2024-11-15T00:47:35,585 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1676): Region close journal for 8072ff2efb6653ab0fbbcdf205cf0e99: Waiting for close lock at 1731631654190Running coprocessor pre-close hooks at 1731631654190Disabling compacts and flushes for region at 1731631654190Disabling writes for close at 1731631654252 (+62 ms)Obtaining lock to block concurrent updates at 1731631654252Preparing flush snapshotting stores in 8072ff2efb6653ab0fbbcdf205cf0e99 at 1731631654252Finished memstore snapshotting TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99., syncing WAL and waiting on mvcc, flushsize=dataSize=89310, getHeapSize=245328, getOffHeapSize=0, getCellsCount=1950 at 1731631654252Flushing stores of TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99. at 1731631654253 (+1 ms)Flushing 8072ff2efb6653ab0fbbcdf205cf0e99/A: creating writer at 1731631654253Flushing mob file 8072ff2efb6653ab0fbbcdf205cf0e99/A: appending metadata at 1731631654259 (+6 ms)Flushing mob file 8072ff2efb6653ab0fbbcdf205cf0e99/A: closing flushed file at 1731631654259Flushing 8072ff2efb6653ab0fbbcdf205cf0e99/A: appending metadata at 1731631654670 (+411 ms)Flushing 8072ff2efb6653ab0fbbcdf205cf0e99/A: closing flushed file at 1731631654671 (+1 ms)Flushing 8072ff2efb6653ab0fbbcdf205cf0e99/B: creating writer at 1731631654684 (+13 ms)Flushing 8072ff2efb6653ab0fbbcdf205cf0e99/B: appending metadata at 1731631654687 (+3 ms)Flushing 8072ff2efb6653ab0fbbcdf205cf0e99/B: closing flushed file at 1731631654687Flushing 8072ff2efb6653ab0fbbcdf205cf0e99/C: creating writer at 1731631655105 (+418 ms)Flushing 8072ff2efb6653ab0fbbcdf205cf0e99/C: appending metadata at 1731631655108 (+3 ms)Flushing 8072ff2efb6653ab0fbbcdf205cf0e99/C: closing flushed file at 1731631655108Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37adca95: reopening flushed file at 1731631655516 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66e2cdec: reopening flushed file at 1731631655520 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7158c53e: reopening flushed file at 1731631655525 (+5 ms)Finished flush of dataSize ~87.22 KB/89310, heapSize ~239.58 KB/245328, currentSize=0 B/0 for 8072ff2efb6653ab0fbbcdf205cf0e99 in 1277ms, sequenceid=467, compaction requested=true at 1731631655530 (+5 ms)Writing region close event to WAL at 1731631655582 (+52 ms)Running coprocessor post-close hooks at 1731631655585 (+3 ms)Closed at 1731631655585 2024-11-15T00:47:35,589 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(157): Closed 8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:35,590 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=8072ff2efb6653ab0fbbcdf205cf0e99, regionState=CLOSED 2024-11-15T00:47:35,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 because future has completed 2024-11-15T00:47:35,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-15T00:47:35,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseRegionProcedure 8072ff2efb6653ab0fbbcdf205cf0e99, server=fb2f6cb7846a,34537,1731631472208 in 1.5540 sec 2024-11-15T00:47:35,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-15T00:47:35,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8072ff2efb6653ab0fbbcdf205cf0e99, UNASSIGN in 1.5580 sec 2024-11-15T00:47:35,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-15T00:47:35,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5610 sec 2024-11-15T00:47:35,597 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631655596"}]},"ts":"1731631655596"} 2024-11-15T00:47:35,598 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:47:35,598 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:47:35,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5670 sec 2024-11-15T00:47:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T00:47:36,179 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:36,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:47:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:36,182 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=154, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T00:47:36,183 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=154, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:36,185 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,200 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C, FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/recovered.edits] 2024-11-15T00:47:36,202 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/60f7abb36bb54bb7a351e1ce862535b3 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/60f7abb36bb54bb7a351e1ce862535b3 2024-11-15T00:47:36,203 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/767a359b7fba4486af4ecd78b79ff4b4 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/767a359b7fba4486af4ecd78b79ff4b4 2024-11-15T00:47:36,204 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/95b3b33f6b9a4575918dda37e473515f to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/95b3b33f6b9a4575918dda37e473515f 2024-11-15T00:47:36,205 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/cf16bf69dc164fceaf8c34de1f3d0956 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/cf16bf69dc164fceaf8c34de1f3d0956 2024-11-15T00:47:36,206 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/fe5d84302287425e85ed4bbd4a2e3fcb to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/A/fe5d84302287425e85ed4bbd4a2e3fcb 2024-11-15T00:47:36,207 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/410697313ece4c0380dc831191ac6faf to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/410697313ece4c0380dc831191ac6faf 2024-11-15T00:47:36,208 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/86e05e65db314f84842081ea4d0b0dbf to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/86e05e65db314f84842081ea4d0b0dbf 2024-11-15T00:47:36,209 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/9fb0d457d3444f058f04c71f6525688e to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/9fb0d457d3444f058f04c71f6525688e 2024-11-15T00:47:36,209 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f37d59623ca142e29ab4b93e32ccb761 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f37d59623ca142e29ab4b93e32ccb761 2024-11-15T00:47:36,210 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f7f64a4d681e4e13bee8a79c57e172cd to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/B/f7f64a4d681e4e13bee8a79c57e172cd 2024-11-15T00:47:36,212 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/12f6704da9de462c935e4f207b4bd100 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/12f6704da9de462c935e4f207b4bd100 2024-11-15T00:47:36,213 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4ea6a70cff054b0fae5899bca2143f22 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4ea6a70cff054b0fae5899bca2143f22 2024-11-15T00:47:36,213 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4f95e6b09cd945d3bb464e1e0f95f703 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/4f95e6b09cd945d3bb464e1e0f95f703 2024-11-15T00:47:36,214 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/7b39f0b0851f43929944083125930827 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/7b39f0b0851f43929944083125930827 2024-11-15T00:47:36,215 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/fa5bc57d87e04fb299646a8e79ed3e35 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/C/fa5bc57d87e04fb299646a8e79ed3e35 2024-11-15T00:47:36,216 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/recovered.edits/470.seqid to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99/recovered.edits/470.seqid 2024-11-15T00:47:36,217 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/default/TestAcidGuarantees/8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,217 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:47:36,217 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:47:36,218 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T00:47:36,220 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150b90835d06084799b09f24ba11e7755a_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150b90835d06084799b09f24ba11e7755a_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,221 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150bfa8e226daf4e08918dfa3f43d95bef_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150bfa8e226daf4e08918dfa3f43d95bef_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,222 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150fcbdea95cce43b6b606e42599dfab62_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150fcbdea95cce43b6b606e42599dfab62_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,223 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111525512ade8fbc4c8db357f3869f97184f_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111525512ade8fbc4c8db357f3869f97184f_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,224 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115516c5b73d73449478d271748e6fdab34_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115516c5b73d73449478d271748e6fdab34_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,225 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111563dfe90759e54415b2af23ebd20cf3ae_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111563dfe90759e54415b2af23ebd20cf3ae_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,226 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157545d0091cbd45f0980e89ec5dd61153_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157545d0091cbd45f0980e89ec5dd61153_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,229 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c919d79d846477dbd15ab4bca25525c_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c919d79d846477dbd15ab4bca25525c_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,230 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159da851d10852421a82d3b973726d7944_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159da851d10852421a82d3b973726d7944_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,231 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e8237fe82df44b9bedf2f45c1ded2a3_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e8237fe82df44b9bedf2f45c1ded2a3_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,232 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159f151065ce6b43e386c510b8e192d533_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159f151065ce6b43e386c510b8e192d533_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,233 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b5975856aa53417fbc576a1a533f252a_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b5975856aa53417fbc576a1a533f252a_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,234 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf3e26395fb0488c9a7710b89535456c_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf3e26395fb0488c9a7710b89535456c_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,235 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c644bca86e714bb1813fe640cbed9bb5_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c644bca86e714bb1813fe640cbed9bb5_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,235 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c671f2b9409f481a9ce1f07f9d274013_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c671f2b9409f481a9ce1f07f9d274013_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,236 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cf36ce9d264e432c8ef90202c0af0bcd_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cf36ce9d264e432c8ef90202c0af0bcd_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,237 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e753c0ddbf254f0a825f1c5423d13df2_8072ff2efb6653ab0fbbcdf205cf0e99 to hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e753c0ddbf254f0a825f1c5423d13df2_8072ff2efb6653ab0fbbcdf205cf0e99 2024-11-15T00:47:36,237 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:47:36,239 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=154, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:36,241 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:47:36,243 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:47:36,244 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=154, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:36,245 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:47:36,245 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631656245"}]},"ts":"9223372036854775807"} 2024-11-15T00:47:36,247 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:47:36,247 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8072ff2efb6653ab0fbbcdf205cf0e99, NAME => 'TestAcidGuarantees,,1731631629929.8072ff2efb6653ab0fbbcdf205cf0e99.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:47:36,247 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:47:36,247 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631656247"}]},"ts":"9223372036854775807"} 2024-11-15T00:47:36,249 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:47:36,250 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=154, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:47:36,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-15T00:47:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T00:47:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:47:36,295 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:47:36,302 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 250), OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1019 (was 1070), ProcessCount=11 (was 11), AvailableMemoryMB=2589 (was 1993) - AvailableMemoryMB LEAK? - 2024-11-15T00:47:36,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T00:47:36,303 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:47:36,303 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:47:36,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:36,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:36,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:47:36,303 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T00:47:36,303 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1866486918, stopped=false 2024-11-15T00:47:36,304 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fb2f6cb7846a,33521,1731631471305 2024-11-15T00:47:36,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T00:47:36,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T00:47:36,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:47:36,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:47:36,314 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:47:36,315 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:47:36,315 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:47:36,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:36,315 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:47:36,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:47:36,316 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fb2f6cb7846a,34537,1731631472208' ***** 2024-11-15T00:47:36,316 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T00:47:36,317 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T00:47:36,317 INFO [RS:0;fb2f6cb7846a:34537 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T00:47:36,317 INFO [RS:0;fb2f6cb7846a:34537 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T00:47:36,317 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(959): stopping server fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:36,317 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T00:47:36,317 INFO [RS:0;fb2f6cb7846a:34537 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fb2f6cb7846a:34537. 2024-11-15T00:47:36,317 DEBUG [RS:0;fb2f6cb7846a:34537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:47:36,317 DEBUG [RS:0;fb2f6cb7846a:34537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:36,317 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T00:47:36,317 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T00:47:36,317 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T00:47:36,317 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T00:47:36,318 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T00:47:36,318 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T00:47:36,318 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:47:36,318 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T00:47:36,318 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T00:47:36,319 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T00:47:36,319 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T00:47:36,319 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T00:47:36,319 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T00:47:36,319 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-15T00:47:36,346 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/info/bf130924b44f4e62b68ff3405d5c1dad is 95, key is TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e./info:/1731631510595/DeleteFamily/seqid=0 2024-11-15T00:47:36,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742494_1670 (size=6514) 2024-11-15T00:47:36,401 INFO [regionserver/fb2f6cb7846a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T00:47:36,519 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:47:36,717 INFO [regionserver/fb2f6cb7846a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T00:47:36,717 INFO [regionserver/fb2f6cb7846a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T00:47:36,719 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:47:36,750 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/info/bf130924b44f4e62b68ff3405d5c1dad 2024-11-15T00:47:36,771 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/ns/497ff197e54c4632a7bd0cec6b058f2b is 93, key is TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e./ns:/1731631510529/DeleteFamily/seqid=0 2024-11-15T00:47:36,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742495_1671 (size=5906) 2024-11-15T00:47:36,919 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:47:37,119 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:47:37,175 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/ns/497ff197e54c4632a7bd0cec6b058f2b 2024-11-15T00:47:37,201 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/rep_barrier/0515ff33edcc45a59959aa166d76943b is 102, key is TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e./rep_barrier:/1731631510529/DeleteFamily/seqid=0 2024-11-15T00:47:37,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742496_1672 (size=6025) 2024-11-15T00:47:37,207 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/rep_barrier/0515ff33edcc45a59959aa166d76943b 2024-11-15T00:47:37,230 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/table/c85093747b594fe4ab51ca80321ab23c is 96, key is TestAcidGuarantees,,1731631477086.c6201d6aebcecefaa2f781f0e095054e./table:/1731631510529/DeleteFamily/seqid=0 2024-11-15T00:47:37,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742497_1673 (size=5936) 2024-11-15T00:47:37,319 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:47:37,520 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:47:37,646 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/table/c85093747b594fe4ab51ca80321ab23c 2024-11-15T00:47:37,681 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/info/bf130924b44f4e62b68ff3405d5c1dad as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/info/bf130924b44f4e62b68ff3405d5c1dad 2024-11-15T00:47:37,690 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/info/bf130924b44f4e62b68ff3405d5c1dad, entries=12, sequenceid=90, filesize=6.4 K 2024-11-15T00:47:37,691 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/ns/497ff197e54c4632a7bd0cec6b058f2b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/ns/497ff197e54c4632a7bd0cec6b058f2b 2024-11-15T00:47:37,720 DEBUG [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:47:37,725 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/ns/497ff197e54c4632a7bd0cec6b058f2b, entries=8, sequenceid=90, filesize=5.8 K 2024-11-15T00:47:37,732 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/rep_barrier/0515ff33edcc45a59959aa166d76943b as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/rep_barrier/0515ff33edcc45a59959aa166d76943b 2024-11-15T00:47:37,738 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/rep_barrier/0515ff33edcc45a59959aa166d76943b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-15T00:47:37,752 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/.tmp/table/c85093747b594fe4ab51ca80321ab23c as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/table/c85093747b594fe4ab51ca80321ab23c 2024-11-15T00:47:37,793 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/table/c85093747b594fe4ab51ca80321ab23c, entries=7, sequenceid=90, filesize=5.8 K 2024-11-15T00:47:37,800 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1481ms, sequenceid=90, compaction requested=false 2024-11-15T00:47:37,901 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-15T00:47:37,905 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T00:47:37,905 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T00:47:37,905 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731631656318Running coprocessor pre-close hooks at 1731631656318Disabling compacts and flushes for region at 1731631656318Disabling writes for close at 1731631656319 (+1 ms)Obtaining lock to block concurrent updates at 1731631656319Preparing flush snapshotting stores in 1588230740 at 1731631656319Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731631656319Flushing stores of hbase:meta,,1.1588230740 at 1731631656320 (+1 ms)Flushing 1588230740/info: creating writer at 1731631656320Flushing 1588230740/info: appending metadata at 1731631656346 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731631656346Flushing 1588230740/ns: creating writer at 1731631656754 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731631656770 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731631656770Flushing 1588230740/rep_barrier: creating writer at 1731631657180 (+410 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731631657200 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731631657200Flushing 1588230740/table: creating writer at 1731631657211 (+11 ms)Flushing 1588230740/table: appending metadata at 1731631657230 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731631657230Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cf9bf5f: reopening flushed file at 1731631657667 (+437 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14b340ad: reopening flushed file at 1731631657691 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76d73a0c: reopening flushed file at 1731631657725 (+34 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48db177d: reopening flushed file at 1731631657738 (+13 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1481ms, sequenceid=90, compaction requested=false at 1731631657800 (+62 ms)Writing region close event to WAL at 1731631657868 (+68 ms)Running coprocessor post-close hooks at 1731631657905 (+37 ms)Closed at 1731631657905 2024-11-15T00:47:37,905 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T00:47:37,922 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(976): stopping server fb2f6cb7846a,34537,1731631472208; all regions closed. 2024-11-15T00:47:37,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741834_1010 (size=25344) 2024-11-15T00:47:37,934 DEBUG [RS:0;fb2f6cb7846a:34537 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/oldWALs 2024-11-15T00:47:37,934 INFO [RS:0;fb2f6cb7846a:34537 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fb2f6cb7846a%2C34537%2C1731631472208.meta:.meta(num 1731631476326) 2024-11-15T00:47:37,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741833_1009 (size=17985910) 2024-11-15T00:47:37,967 DEBUG [RS:0;fb2f6cb7846a:34537 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/oldWALs 2024-11-15T00:47:37,967 INFO [RS:0;fb2f6cb7846a:34537 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fb2f6cb7846a%2C34537%2C1731631472208:(num 1731631475893) 2024-11-15T00:47:37,967 DEBUG [RS:0;fb2f6cb7846a:34537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:47:37,967 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T00:47:37,967 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T00:47:37,967 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.ChoreService(370): Chore service for: regionserver/fb2f6cb7846a:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T00:47:37,968 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T00:47:37,968 INFO [regionserver/fb2f6cb7846a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T00:47:37,969 INFO [RS:0;fb2f6cb7846a:34537 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34537 2024-11-15T00:47:37,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T00:47:37,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fb2f6cb7846a,34537,1731631472208 2024-11-15T00:47:37,986 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T00:47:37,999 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fb2f6cb7846a,34537,1731631472208] 2024-11-15T00:47:38,009 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fb2f6cb7846a,34537,1731631472208 already deleted, retry=false 2024-11-15T00:47:38,009 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fb2f6cb7846a,34537,1731631472208 expired; onlineServers=0 2024-11-15T00:47:38,009 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fb2f6cb7846a,33521,1731631471305' ***** 2024-11-15T00:47:38,009 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T00:47:38,010 INFO [M:0;fb2f6cb7846a:33521 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T00:47:38,010 INFO [M:0;fb2f6cb7846a:33521 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T00:47:38,010 DEBUG [M:0;fb2f6cb7846a:33521 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T00:47:38,010 DEBUG [M:0;fb2f6cb7846a:33521 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T00:47:38,010 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.large.0-1731631475356 {}] cleaner.HFileCleaner(306): Exit Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.large.0-1731631475356,5,FailOnTimeoutGroup] 2024-11-15T00:47:38,010 INFO [M:0;fb2f6cb7846a:33521 {}] hbase.ChoreService(370): Chore service for: master/fb2f6cb7846a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T00:47:38,010 INFO [M:0;fb2f6cb7846a:33521 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T00:47:38,010 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.small.0-1731631475368 {}] cleaner.HFileCleaner(306): Exit Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.small.0-1731631475368,5,FailOnTimeoutGroup] 2024-11-15T00:47:38,010 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T00:47:38,010 DEBUG [M:0;fb2f6cb7846a:33521 {}] master.HMaster(1795): Stopping service threads 2024-11-15T00:47:38,010 INFO [M:0;fb2f6cb7846a:33521 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T00:47:38,011 INFO [M:0;fb2f6cb7846a:33521 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:47:38,011 ERROR [M:0;fb2f6cb7846a:33521 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T00:47:38,012 INFO [M:0;fb2f6cb7846a:33521 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:47:38,012 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T00:47:38,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T00:47:38,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:47:38,026 DEBUG [M:0;fb2f6cb7846a:33521 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-15T00:47:38,026 DEBUG [M:0;fb2f6cb7846a:33521 {}] master.ActiveMasterManager(353): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-15T00:47:38,030 INFO [M:0;fb2f6cb7846a:33521 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/.lastflushedseqids 2024-11-15T00:47:38,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742498_1674 (size=72) 2024-11-15T00:47:38,080 INFO [M:0;fb2f6cb7846a:33521 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:47:38,080 INFO [M:0;fb2f6cb7846a:33521 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T00:47:38,080 DEBUG [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T00:47:38,080 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:47:38,080 DEBUG [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:47:38,080 DEBUG [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T00:47:38,080 DEBUG [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:47:38,081 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=734.32 KB heapSize=897.15 KB 2024-11-15T00:47:38,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:47:38,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34537-0x1013bf352ec0001, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:47:38,101 INFO [RS:0;fb2f6cb7846a:34537 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T00:47:38,102 INFO [RS:0;fb2f6cb7846a:34537 {}] regionserver.HRegionServer(1031): Exiting; stopping=fb2f6cb7846a,34537,1731631472208; zookeeper connection closed. 2024-11-15T00:47:38,111 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6053c025 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6053c025 2024-11-15T00:47:38,112 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T00:47:38,118 DEBUG [M:0;fb2f6cb7846a:33521 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ce3a006c2a224ecaa650af6895e33539 is 82, key is hbase:meta,,1/info:regioninfo/1731631476510/Put/seqid=0 2024-11-15T00:47:38,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742499_1675 (size=5672) 2024-11-15T00:47:38,165 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2009 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ce3a006c2a224ecaa650af6895e33539 2024-11-15T00:47:38,241 DEBUG [M:0;fb2f6cb7846a:33521 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/71f8c728078d4230beb8a475bcdb0219 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x85/proc:d/1731631633019/Put/seqid=0 2024-11-15T00:47:38,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742500_1676 (size=42556) 2024-11-15T00:47:38,660 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=733.77 KB at sequenceid=2009 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/71f8c728078d4230beb8a475bcdb0219 2024-11-15T00:47:38,665 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 71f8c728078d4230beb8a475bcdb0219 2024-11-15T00:47:38,684 DEBUG [M:0;fb2f6cb7846a:33521 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6652ea89e902404ea78f747649116eee is 69, key is fb2f6cb7846a,34537,1731631472208/rs:state/1731631475522/Put/seqid=0 2024-11-15T00:47:38,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073742501_1677 (size=5156) 2024-11-15T00:47:38,693 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2009 (bloomFilter=true), to=hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6652ea89e902404ea78f747649116eee 2024-11-15T00:47:38,697 DEBUG [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ce3a006c2a224ecaa650af6895e33539 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ce3a006c2a224ecaa650af6895e33539 2024-11-15T00:47:38,701 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ce3a006c2a224ecaa650af6895e33539, entries=8, sequenceid=2009, filesize=5.5 K 2024-11-15T00:47:38,701 DEBUG [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/71f8c728078d4230beb8a475bcdb0219 as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/71f8c728078d4230beb8a475bcdb0219 2024-11-15T00:47:38,706 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 71f8c728078d4230beb8a475bcdb0219 2024-11-15T00:47:38,706 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/71f8c728078d4230beb8a475bcdb0219, entries=154, sequenceid=2009, filesize=41.6 K 2024-11-15T00:47:38,707 DEBUG [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6652ea89e902404ea78f747649116eee as hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6652ea89e902404ea78f747649116eee 2024-11-15T00:47:38,712 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42881/user/jenkins/test-data/b23ebd44-cf12-25e5-4e6c-eefd68cf0403/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6652ea89e902404ea78f747649116eee, entries=1, sequenceid=2009, filesize=5.0 K 2024-11-15T00:47:38,713 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegion(3140): Finished flush of dataSize ~734.32 KB/751945, heapSize ~896.85 KB/918376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 633ms, sequenceid=2009, compaction requested=false 2024-11-15T00:47:38,764 INFO [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:47:38,764 DEBUG [M:0;fb2f6cb7846a:33521 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731631658080Disabling compacts and flushes for region at 1731631658080Disabling writes for close at 1731631658080Obtaining lock to block concurrent updates at 1731631658081 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731631658081Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=751945, getHeapSize=918616, getOffHeapSize=0, getCellsCount=2188 at 1731631658081Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731631658082 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731631658082Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731631658118 (+36 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731631658118Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731631658207 (+89 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731631658240 (+33 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731631658240Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731631658665 (+425 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731631658683 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731631658683Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@409dc288: reopening flushed file at 1731631658696 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d0ffd5c: reopening flushed file at 1731631658701 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4131405e: reopening flushed file at 1731631658706 (+5 ms)Finished flush of dataSize ~734.32 KB/751945, heapSize ~896.85 KB/918376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 633ms, sequenceid=2009, compaction requested=false at 1731631658714 (+8 ms)Writing region close event to WAL at 1731631658764 (+50 ms)Closed at 1731631658764 2024-11-15T00:47:38,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38979 is added to blk_1073741830_1006 (size=883665) 2024-11-15T00:47:38,774 INFO [M:0;fb2f6cb7846a:33521 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T00:47:38,774 INFO [M:0;fb2f6cb7846a:33521 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33521 2024-11-15T00:47:38,774 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T00:47:38,774 INFO [M:0;fb2f6cb7846a:33521 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T00:47:38,899 INFO [M:0;fb2f6cb7846a:33521 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T00:47:38,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:47:38,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1013bf352ec0000, quorum=127.0.0.1:62640, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:47:38,969 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T00:47:38,971 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T00:47:38,972 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T00:47:38,972 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T00:47:38,972 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/hadoop.log.dir/,STOPPED} 2024-11-15T00:47:38,990 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T00:47:38,990 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T00:47:38,993 WARN [BP-1642220532-172.17.0.3-1731631466348 heartbeating to localhost/127.0.0.1:42881 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T00:47:38,993 WARN [BP-1642220532-172.17.0.3-1731631466348 heartbeating to localhost/127.0.0.1:42881 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1642220532-172.17.0.3-1731631466348 (Datanode Uuid 7fa1af05-92e5-4b5f-9f70-a44c27aaea50) service to localhost/127.0.0.1:42881 2024-11-15T00:47:38,999 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/cluster_c94eddf5-b65f-7607-3f6e-7f317580d804/data/data1/current/BP-1642220532-172.17.0.3-1731631466348 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T00:47:38,999 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/cluster_c94eddf5-b65f-7607-3f6e-7f317580d804/data/data2/current/BP-1642220532-172.17.0.3-1731631466348 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T00:47:38,999 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T00:47:39,038 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T00:47:39,051 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T00:47:39,051 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T00:47:39,053 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T00:47:39,053 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/df0864d2-704d-65d2-cf27-72c373e54a7d/hadoop.log.dir/,STOPPED} 2024-11-15T00:47:39,176 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T00:47:39,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-15T00:47:41,087 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:47:41,091 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null